2024-09-14T03:29:39.670 INFO:root:teuthology version: 1.2.2.dev2+g3752d38 2024-09-14T03:29:39.693 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-09-14T03:29:39.785 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2024-09-14_03:08:15-upgrade-main-distro-default-smithi/7904542 branch: main description: upgrade/cephfs/nofs/{bluestore-bitmap centos_9.stream conf/{client mds mgr mon osd} no-mds-cluster overrides/{ignorelist_health ignorelist_wrongly_marked_down pg-warn pg_health} tasks/{0-from/squid 1-upgrade}} email: ceph-qa@ceph.com first_in_suite: false job_id: '7904542' kernel: kdb: 1 sha1: distro last_in_suite: false machine_type: smithi meta: - desc: 'install ceph/squid latest ' name: teuthology-2024-09-14_03:08:15-upgrade-main-distro-default-smithi no_nested_subset: false openstack: - volumes: count: 4 size: 10 os_type: centos os_version: 9.stream overrides: admin_socket: branch: main ceph: cluster-conf: client: client mount timeout: 600 debug client: 20 debug ms: 1 rados mon op timeout: 900 rados osd op timeout: 900 mds: debug mds: 20 debug mds balancer: 20 debug ms: 2 mds debug frag: true mds debug scatterstat: true mds op complaint time: 180 mds verify scatter: true osd op complaint time: 180 rados mon op timeout: 900 rados osd op timeout: 900 mon: mon op complaint time: 120 mon warn on pool no app: false osd: osd op complaint time: 180 conf: global: bluestore warn on legacy statfs: false bluestore warn on no per pool omap: false mon pg warn min per osd: 0 mgr: client mount timeout: 30 debug client: 20 debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 mon down mkfs grace: 300 mon warn on osd down out interval zero: false osd: bdev async discard: true bdev enable discard: true bluestore allocator: bitmap bluestore block size: 96636764160 bluestore fsck on mount: true debug bluefs: 1/20 debug bluestore: 1/20 debug ms: 1 debug osd: 20 debug rocksdb: 4/10 mon osd backfillfull_ratio: 0.85 mon osd full ratio: 0.9 mon osd nearfull ratio: 0.8 osd failsafe full ratio: 0.95 osd objectstore: bluestore flavor: default fs: xfs log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) - FS_DEGRADED - fs.*is degraded - filesystem is degraded - FS_INLINE_DATA_DEPRECATED - FS_WITH_FAILED_MDS - MDS_ALL_DOWN - filesystem is offline - MDS_DAMAGE - MDS_DEGRADED - MDS_FAILED - MDS_INSUFFICIENT_STANDBY - insufficient standby MDS daemons available - MDS_UP_LESS_THAN_MAX - online, but wants - filesystem is online with fewer MDS than max_mds - POOL_APP_NOT_ENABLED - do not have an application enabled - overall HEALTH_ - Replacing daemon - deprecated feature inline_data - overall HEALTH_ - \(OSD_DOWN\) - \(OSD_ - but it is still running - is not responding - PG_AVAILABILITY - PG_DEGRADED - Reduced data availability - Degraded data redundancy - scrub mismatch - ScrubResult - wrongly marked - \(POOL_APP_NOT_ENABLED\) - \(SLOW_OPS\) - overall HEALTH_ - \(MON_MSGR2_NOT_ENABLED\) - slow request sha1: a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 ceph-deploy: bluestore: true conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} osd: bdev async discard: true bdev enable discard: true bluestore block size: 96636764160 bluestore fsck on mount: true debug bluefs: 1/20 debug bluestore: 1/20 debug rocksdb: 4/10 mon osd backfillfull_ratio: 0.85 mon osd full ratio: 0.9 mon osd nearfull ratio: 0.8 osd failsafe full ratio: 0.95 osd objectstore: bluestore fs: xfs install: ceph: flavor: default sha1: a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 selinux: allowlist: - scontext=system_u:system_r:logrotate_t:s0 thrashosds: bdev_inject_crash: 2 bdev_inject_crash_probability: 0.5 workunit: branch: main sha1: 1c941a66c1b420259b8fc5ac19e792dbbf60aded owner: scheduled_teuthology@teuthology priority: 850 repo: https://git.ceph.com/ceph.git roles: - - mon.a - mon.b - mon.c - mgr.x - mgr.y - osd.0 - osd.1 - osd.2 - osd.3 seed: 8038 sha1: a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 sleep_before_teardown: 0 subset: 31/32 suite: upgrade suite_branch: main suite_path: /home/teuthworker/src/git.ceph.com_ceph_1c941a66c1b420259b8fc5ac19e792dbbf60aded/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph.git suite_sha1: 1c941a66c1b420259b8fc5ac19e792dbbf60aded targets: smithi083.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBIZoFnW86DoNtGbH4n6H+OCmegCYz4pyfROCf0vMbq7KBlfWRiL2WlsvNlIzYA8hvnHwFYRtAzMd1W6phaUAILc= tasks: - install: branch: squid exclude_packages: - librados3 - ceph-mgr-dashboard - ceph-mgr-diskprediction-local - ceph-mgr-rook - ceph-mgr-cephadm - cephadm - ceph-volume extra_packages: - librados2 - print: '**** done installing squid' - ceph: conf: global: mon warn on pool no app: false ms bind msgr2: false log-ignorelist: - overall HEALTH_ - \(FS_ - \(MDS_ - \(OSD_ - \(MON_DOWN\) - \(CACHE_POOL_ - \(POOL_ - \(MGR_DOWN\) - \(PG_ - \(SMALLER_PGP_NUM\) - Monitor daemon marked osd - Behind on trimming - Manager daemon - exec: osd.0: - ceph osd set-require-min-compat-client squid - print: '**** done ceph' - print: '*** upgrading, no cephfs present' - exec: mon.a: - ceph fs dump - install.upgrade: mon.a: null - print: '**** done install.upgrade' - ceph.restart: daemons: - mon.* - mgr.* mon-health-to-clog: false wait-for-healthy: false - ceph.healthy: null - ceph.restart: daemons: - osd.* wait-for-healthy: false wait-for-osds-up: true - exec: mon.a: - ceph versions - ceph osd dump -f json-pretty - ceph fs dump - ceph osd require-osd-release squid - for f in `ceph osd pool ls` ; do ceph osd pool set $f pg_autoscale_mode off ; done - ceph.healthy: null - print: '**** done ceph.restart' teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a timestamp: 2024-09-14_03:08:15 tube: smithi user: teuthology verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.3401856 2024-09-14T03:29:39.786 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph_1c941a66c1b420259b8fc5ac19e792dbbf60aded/qa; will attempt to use it 2024-09-14T03:29:39.786 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph_1c941a66c1b420259b8fc5ac19e792dbbf60aded/qa/tasks 2024-09-14T03:29:39.787 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-09-14T03:29:39.788 INFO:teuthology.task.internal:Checking packages... 2024-09-14T03:29:39.810 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash 'a2782315255a3e7d5f6cdc9eb35785187ff6ddf8' 2024-09-14T03:29:39.810 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-14T03:29:39.810 INFO:teuthology.packaging:ref: None 2024-09-14T03:29:39.811 INFO:teuthology.packaging:tag: None 2024-09-14T03:29:39.811 INFO:teuthology.packaging:branch: main 2024-09-14T03:29:39.811 INFO:teuthology.packaging:sha1: a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T03:29:39.811 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=main 2024-09-14T03:29:40.070 INFO:teuthology.task.internal:Found packages for ceph version 19.3.0-4926.g1c941a66 2024-09-14T03:29:40.072 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-09-14T03:29:40.078 INFO:teuthology.task.internal:no buildpackages task found 2024-09-14T03:29:40.078 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-09-14T03:29:40.130 INFO:teuthology.task.internal:Saving configuration 2024-09-14T03:29:40.148 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-09-14T03:29:40.163 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-09-14T03:29:40.188 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi083.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-09-14_03:08:15-upgrade-main-distro-default-smithi/7904542', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-09-14 03:24:49.570373', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBIZoFnW86DoNtGbH4n6H+OCmegCYz4pyfROCf0vMbq7KBlfWRiL2WlsvNlIzYA8hvnHwFYRtAzMd1W6phaUAILc='} 2024-09-14T03:29:40.188 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-09-14T03:29:40.201 INFO:teuthology.task.internal:roles: ubuntu@smithi083.front.sepia.ceph.com - ['mon.a', 'mon.b', 'mon.c', 'mgr.x', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3'] 2024-09-14T03:29:40.201 INFO:teuthology.run_tasks:Running task console_log... 2024-09-14T03:29:40.258 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7ff545d43ac0>, signals=[15]) 2024-09-14T03:29:40.258 INFO:teuthology.run_tasks:Running task internal.connect... 2024-09-14T03:29:40.270 INFO:teuthology.task.internal:Opening connections... 2024-09-14T03:29:40.270 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi083.front.sepia.ceph.com 2024-09-14T03:29:40.271 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi083.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-14T03:29:40.347 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-09-14T03:29:40.356 DEBUG:teuthology.orchestra.run.smithi083:> uname -m 2024-09-14T03:29:40.434 INFO:teuthology.orchestra.run.smithi083.stdout:x86_64 2024-09-14T03:29:40.434 DEBUG:teuthology.orchestra.run.smithi083:> cat /etc/os-release 2024-09-14T03:29:40.491 INFO:teuthology.orchestra.run.smithi083.stdout:NAME="CentOS Stream" 2024-09-14T03:29:40.491 INFO:teuthology.orchestra.run.smithi083.stdout:VERSION="9" 2024-09-14T03:29:40.491 INFO:teuthology.orchestra.run.smithi083.stdout:ID="centos" 2024-09-14T03:29:40.491 INFO:teuthology.orchestra.run.smithi083.stdout:ID_LIKE="rhel fedora" 2024-09-14T03:29:40.491 INFO:teuthology.orchestra.run.smithi083.stdout:VERSION_ID="9" 2024-09-14T03:29:40.491 INFO:teuthology.orchestra.run.smithi083.stdout:PLATFORM_ID="platform:el9" 2024-09-14T03:29:40.491 INFO:teuthology.orchestra.run.smithi083.stdout:PRETTY_NAME="CentOS Stream 9" 2024-09-14T03:29:40.491 INFO:teuthology.orchestra.run.smithi083.stdout:ANSI_COLOR="0;31" 2024-09-14T03:29:40.492 INFO:teuthology.orchestra.run.smithi083.stdout:LOGO="fedora-logo-icon" 2024-09-14T03:29:40.492 INFO:teuthology.orchestra.run.smithi083.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-09-14T03:29:40.492 INFO:teuthology.orchestra.run.smithi083.stdout:HOME_URL="https://centos.org/" 2024-09-14T03:29:40.492 INFO:teuthology.orchestra.run.smithi083.stdout:BUG_REPORT_URL="https://issues.redhat.com/" 2024-09-14T03:29:40.492 INFO:teuthology.orchestra.run.smithi083.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-09-14T03:29:40.492 INFO:teuthology.orchestra.run.smithi083.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-09-14T03:29:40.493 INFO:teuthology.lock.ops:Updating smithi083.front.sepia.ceph.com on lock server 2024-09-14T03:29:40.514 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-09-14T03:29:40.524 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-09-14T03:29:40.549 INFO:teuthology.task.internal:Checking for old test directory... 2024-09-14T03:29:40.549 DEBUG:teuthology.orchestra.run.smithi083:> test '!' -e /home/ubuntu/cephtest 2024-09-14T03:29:40.566 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-09-14T03:29:40.582 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-09-14T03:29:40.582 DEBUG:teuthology.orchestra.run.smithi083:> test -z $(ls -A /var/lib/ceph) 2024-09-14T03:29:40.628 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-09-14T03:29:40.664 INFO:teuthology.run_tasks:Running task kernel... 2024-09-14T03:29:40.685 INFO:teuthology.task.kernel:normalize config orig: {'kdb': 1, 'sha1': 'distro'} 2024-09-14T03:29:40.685 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': 1, 'sha1': 'distro'}}, timeout 300 2024-09-14T03:29:40.685 DEBUG:teuthology.orchestra.run.smithi083:> test -f /run/.containerenv -o -f /.dockerenv 2024-09-14T03:29:40.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T03:29:40.702 DEBUG:teuthology.orchestra.run.smithi083:> uname -r 2024-09-14T03:29:40.761 INFO:teuthology.orchestra.run.smithi083.stdout:5.14.0-503.el9.x86_64 2024-09-14T03:29:40.761 INFO:teuthology.task.kernel:Running kernel on smithi083: 5.14.0-503.el9.x86_64 2024-09-14T03:29:40.761 DEBUG:teuthology.orchestra.run.smithi083:> sudo yum install -y kernel 2024-09-14T03:29:43.218 INFO:teuthology.orchestra.run.smithi083.stdout:CentOS Stream 9 - BaseOS 11 MB/s | 8.2 MB 00:00 2024-09-14T03:29:45.027 INFO:teuthology.orchestra.run.smithi083.stdout:CentOS Stream 9 - AppStream 30 MB/s | 20 MB 00:00 2024-09-14T03:29:49.395 INFO:teuthology.orchestra.run.smithi083.stdout:CentOS Stream 9 - CRB 11 MB/s | 6.5 MB 00:00 2024-09-14T03:29:51.596 INFO:teuthology.orchestra.run.smithi083.stdout:CentOS Stream 9 - Extras packages 19 kB/s | 19 kB 00:01 2024-09-14T03:29:52.483 INFO:teuthology.orchestra.run.smithi083.stdout:Extra Packages for Enterprise Linux 30 MB/s | 23 MB 00:00 2024-09-14T03:29:57.847 INFO:teuthology.orchestra.run.smithi083.stdout:lab-extras 45 kB/s | 1.7 kB 00:00 2024-09-14T03:29:59.870 INFO:teuthology.orchestra.run.smithi083.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-09-14T03:29:59.871 INFO:teuthology.orchestra.run.smithi083.stdout:Package kernel-5.14.0-503.el9.x86_64 is already installed. 2024-09-14T03:29:59.964 INFO:teuthology.orchestra.run.smithi083.stdout:Dependencies resolved. 2024-09-14T03:29:59.979 INFO:teuthology.orchestra.run.smithi083.stdout:================================================================================ 2024-09-14T03:29:59.980 INFO:teuthology.orchestra.run.smithi083.stdout: Package Architecture Version Repository Size 2024-09-14T03:29:59.980 INFO:teuthology.orchestra.run.smithi083.stdout:================================================================================ 2024-09-14T03:29:59.980 INFO:teuthology.orchestra.run.smithi083.stdout:Installing: 2024-09-14T03:29:59.980 INFO:teuthology.orchestra.run.smithi083.stdout: kernel x86_64 5.14.0-505.el9 baseos 23 k 2024-09-14T03:29:59.980 INFO:teuthology.orchestra.run.smithi083.stdout:Installing dependencies: 2024-09-14T03:29:59.980 INFO:teuthology.orchestra.run.smithi083.stdout: kernel-core x86_64 5.14.0-505.el9 baseos 16 M 2024-09-14T03:29:59.980 INFO:teuthology.orchestra.run.smithi083.stdout: kernel-modules x86_64 5.14.0-505.el9 baseos 35 M 2024-09-14T03:29:59.980 INFO:teuthology.orchestra.run.smithi083.stdout: kernel-modules-core x86_64 5.14.0-505.el9 baseos 29 M 2024-09-14T03:29:59.980 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:29:59.980 INFO:teuthology.orchestra.run.smithi083.stdout:Transaction Summary 2024-09-14T03:29:59.980 INFO:teuthology.orchestra.run.smithi083.stdout:================================================================================ 2024-09-14T03:29:59.981 INFO:teuthology.orchestra.run.smithi083.stdout:Install 4 Packages 2024-09-14T03:29:59.981 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:29:59.981 INFO:teuthology.orchestra.run.smithi083.stdout:Total download size: 79 M 2024-09-14T03:29:59.981 INFO:teuthology.orchestra.run.smithi083.stdout:Installed size: 126 M 2024-09-14T03:29:59.981 INFO:teuthology.orchestra.run.smithi083.stdout:Downloading Packages: 2024-09-14T03:30:00.324 INFO:teuthology.orchestra.run.smithi083.stdout:(1/4): kernel-5.14.0-505.el9.x86_64.rpm 78 kB/s | 23 kB 00:00 2024-09-14T03:30:16.798 INFO:teuthology.orchestra.run.smithi083.stdout:(2/4): kernel-core-5.14.0-505.el9.x86_64.rpm 956 kB/s | 16 MB 00:16 2024-09-14T03:30:18.665 INFO:teuthology.orchestra.run.smithi083.stdout:(3/4): kernel-modules-core-5.14.0-505.el9.x86_6 1.6 MB/s | 29 MB 00:18 2024-09-14T03:30:19.414 INFO:teuthology.orchestra.run.smithi083.stdout:(4/4): kernel-modules-5.14.0-505.el9.x86_64.rpm 1.8 MB/s | 35 MB 00:19 2024-09-14T03:30:19.415 INFO:teuthology.orchestra.run.smithi083.stdout:-------------------------------------------------------------------------------- 2024-09-14T03:30:19.415 INFO:teuthology.orchestra.run.smithi083.stdout:Total 4.1 MB/s | 79 MB 00:19 2024-09-14T03:30:20.107 INFO:teuthology.orchestra.run.smithi083.stdout:Running transaction check 2024-09-14T03:30:20.485 INFO:teuthology.orchestra.run.smithi083.stdout:Transaction check succeeded. 2024-09-14T03:30:20.485 INFO:teuthology.orchestra.run.smithi083.stdout:Running transaction test 2024-09-14T03:30:21.446 INFO:teuthology.orchestra.run.smithi083.stdout:Transaction test succeeded. 2024-09-14T03:30:21.446 INFO:teuthology.orchestra.run.smithi083.stdout:Running transaction 2024-09-14T03:30:23.898 INFO:teuthology.orchestra.run.smithi083.stdout: Preparing : 1/1 2024-09-14T03:30:24.621 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : kernel-modules-core-5.14.0-505.el9.x86_64 1/4 2024-09-14T03:30:24.701 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : kernel-core-5.14.0-505.el9.x86_64 2/4 2024-09-14T03:30:27.471 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: kernel-core-5.14.0-505.el9.x86_64 2/4 2024-09-14T03:30:27.595 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : kernel-modules-5.14.0-505.el9.x86_64 3/4 2024-09-14T03:30:32.464 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: kernel-modules-5.14.0-505.el9.x86_64 3/4 2024-09-14T03:30:32.587 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : kernel-5.14.0-505.el9.x86_64 4/4 2024-09-14T03:30:37.412 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: kernel-modules-core-5.14.0-505.el9.x86_64 4/4 2024-09-14T03:31:15.519 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: kernel-core-5.14.0-505.el9.x86_64 4/4 2024-09-14T03:31:15.558 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: kernel-modules-5.14.0-505.el9.x86_64 4/4 2024-09-14T03:31:15.896 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: kernel-5.14.0-505.el9.x86_64 4/4 2024-09-14T03:31:15.896 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : kernel-5.14.0-505.el9.x86_64 1/4 2024-09-14T03:31:15.896 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : kernel-core-5.14.0-505.el9.x86_64 2/4 2024-09-14T03:31:15.896 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : kernel-modules-5.14.0-505.el9.x86_64 3/4 2024-09-14T03:31:16.748 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : kernel-modules-core-5.14.0-505.el9.x86_64 4/4 2024-09-14T03:31:16.748 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:31:16.748 INFO:teuthology.orchestra.run.smithi083.stdout:Installed: 2024-09-14T03:31:16.748 INFO:teuthology.orchestra.run.smithi083.stdout: kernel-5.14.0-505.el9.x86_64 2024-09-14T03:31:16.748 INFO:teuthology.orchestra.run.smithi083.stdout: kernel-core-5.14.0-505.el9.x86_64 2024-09-14T03:31:16.749 INFO:teuthology.orchestra.run.smithi083.stdout: kernel-modules-5.14.0-505.el9.x86_64 2024-09-14T03:31:16.749 INFO:teuthology.orchestra.run.smithi083.stdout: kernel-modules-core-5.14.0-505.el9.x86_64 2024-09-14T03:31:16.749 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:31:16.749 INFO:teuthology.orchestra.run.smithi083.stdout:Complete! 2024-09-14T03:31:17.119 DEBUG:teuthology.orchestra.run.smithi083:> rpm -q kernel | sort -rV | head -n 1 2024-09-14T03:31:17.212 INFO:teuthology.orchestra.run.smithi083.stdout:kernel-5.14.0-505.el9.x86_64 2024-09-14T03:31:17.213 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-505.el9.x86_64 2024-09-14T03:31:17.213 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-503.el9.x86_64 Expected: 5.14.0-505.el9.x86_64 2024-09-14T03:31:17.213 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-09-14T03:31:17.213 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-09-14T03:31:17.213 INFO:teuthology.task.kernel:Installing distro kernel on mon.a... 2024-09-14T03:31:17.213 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi083.front.sepia.ceph.com, path=None, version=distro) 2024-09-14T03:31:17.214 DEBUG:teuthology.orchestra.run.smithi083:> rpm -q kernel | sort -rV | head -n 1 2024-09-14T03:31:17.272 INFO:teuthology.orchestra.run.smithi083.stdout:kernel-5.14.0-505.el9.x86_64 2024-09-14T03:31:17.273 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-505.el9.x86_64 2024-09-14T03:31:17.273 DEBUG:teuthology.orchestra.run.smithi083:> sudo rpm -qi grub2-tools 2024-09-14T03:31:17.305 INFO:teuthology.orchestra.run.smithi083.stdout:Name : grub2-tools 2024-09-14T03:31:17.305 INFO:teuthology.orchestra.run.smithi083.stdout:Epoch : 1 2024-09-14T03:31:17.305 INFO:teuthology.orchestra.run.smithi083.stdout:Version : 2.06 2024-09-14T03:31:17.305 INFO:teuthology.orchestra.run.smithi083.stdout:Release : 82.el9 2024-09-14T03:31:17.305 INFO:teuthology.orchestra.run.smithi083.stdout:Architecture: x86_64 2024-09-14T03:31:17.306 INFO:teuthology.orchestra.run.smithi083.stdout:Install Date: Wed 28 Aug 2024 09:06:27 PM UTC 2024-09-14T03:31:17.306 INFO:teuthology.orchestra.run.smithi083.stdout:Group : Unspecified 2024-09-14T03:31:17.306 INFO:teuthology.orchestra.run.smithi083.stdout:Size : 8274670 2024-09-14T03:31:17.306 INFO:teuthology.orchestra.run.smithi083.stdout:License : GPLv3+ 2024-09-14T03:31:17.306 INFO:teuthology.orchestra.run.smithi083.stdout:Signature : RSA/SHA256, Mon 01 Jul 2024 07:46:42 AM UTC, Key ID 05b555b38483c65d 2024-09-14T03:31:17.306 INFO:teuthology.orchestra.run.smithi083.stdout:Source RPM : grub2-2.06-82.el9.src.rpm 2024-09-14T03:31:17.306 INFO:teuthology.orchestra.run.smithi083.stdout:Build Date : Fri 28 Jun 2024 09:24:44 AM UTC 2024-09-14T03:31:17.306 INFO:teuthology.orchestra.run.smithi083.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-09-14T03:31:17.306 INFO:teuthology.orchestra.run.smithi083.stdout:Packager : builder@centos.org 2024-09-14T03:31:17.306 INFO:teuthology.orchestra.run.smithi083.stdout:Vendor : CentOS 2024-09-14T03:31:17.306 INFO:teuthology.orchestra.run.smithi083.stdout:URL : http://www.gnu.org/software/grub/ 2024-09-14T03:31:17.306 INFO:teuthology.orchestra.run.smithi083.stdout:Summary : Support tools for GRUB. 2024-09-14T03:31:17.306 INFO:teuthology.orchestra.run.smithi083.stdout:Description : 2024-09-14T03:31:17.306 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:31:17.307 INFO:teuthology.orchestra.run.smithi083.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-09-14T03:31:17.307 INFO:teuthology.orchestra.run.smithi083.stdout:customizable bootloader with modular architecture. It supports a rich 2024-09-14T03:31:17.307 INFO:teuthology.orchestra.run.smithi083.stdout:variety of kernel formats, file systems, computer architectures and 2024-09-14T03:31:17.307 INFO:teuthology.orchestra.run.smithi083.stdout:hardware devices. 2024-09-14T03:31:17.307 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:31:17.307 INFO:teuthology.orchestra.run.smithi083.stdout:This subpackage provides tools for support of all platforms. 2024-09-14T03:31:17.308 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-09-14T03:31:17.308 INFO:teuthology.task.kernel:Updating grub on smithi083 to boot 5.14.0-505.el9.x86_64 2024-09-14T03:31:17.308 DEBUG:teuthology.orchestra.run.smithi083:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-09-14T03:31:17.975 INFO:teuthology.orchestra.run.smithi083.stderr:Generating grub configuration file ... 2024-09-14T03:31:19.240 INFO:teuthology.orchestra.run.smithi083.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-09-14T03:31:19.261 INFO:teuthology.orchestra.run.smithi083.stderr:done 2024-09-14T03:31:19.261 DEBUG:teuthology.orchestra.run.smithi083:> mktemp 2024-09-14T03:31:19.277 INFO:teuthology.orchestra.run.smithi083.stdout:/tmp/tmp.ke9OWnJzc0 2024-09-14T03:31:19.277 DEBUG:teuthology.orchestra.run.smithi083:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.ke9OWnJzc0 2024-09-14T03:31:19.343 DEBUG:teuthology.orchestra.run.smithi083:> sudo chmod 0666 /tmp/tmp.ke9OWnJzc0 2024-09-14T03:31:19.479 DEBUG:teuthology.orchestra.remote:smithi083:/tmp/tmp.ke9OWnJzc0 is 6KB 2024-09-14T03:31:19.529 DEBUG:teuthology.orchestra.run.smithi083:> rm -fr /tmp/tmp.ke9OWnJzc0 2024-09-14T03:31:19.553 DEBUG:teuthology.orchestra.run.smithi083:> sudo /bin/ls /boot/loader/entries || true 2024-09-14T03:31:19.616 INFO:teuthology.orchestra.run.smithi083.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-0-rescue.conf 2024-09-14T03:31:19.616 INFO:teuthology.orchestra.run.smithi083.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-239.el9.x86_64.conf 2024-09-14T03:31:19.616 INFO:teuthology.orchestra.run.smithi083.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-503.el9.x86_64.conf 2024-09-14T03:31:19.616 INFO:teuthology.orchestra.run.smithi083.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-505.el9.x86_64.conf 2024-09-14T03:31:19.618 DEBUG:teuthology.orchestra.run.smithi083:> sudo grub2-set-default fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-505.el9.x86_64 2024-09-14T03:31:19.781 DEBUG:teuthology.orchestra.run.smithi083:> sudo shutdown -r now 2024-09-14T03:31:49.813 INFO:teuthology.task.kernel:Checking client mon.a for new kernel version... 2024-09-14T03:31:49.814 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi083.front.sepia.ceph.com' 2024-09-14T03:31:49.814 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi083.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-14T03:32:49.873 DEBUG:teuthology.orchestra.remote:timed out 2024-09-14T03:32:58.881 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi083.front.sepia.ceph.com' 2024-09-14T03:32:58.883 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi083.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-14T03:32:59.352 DEBUG:teuthology.orchestra.run.smithi083:> true 2024-09-14T03:32:59.748 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi083.front.sepia.ceph.com' 2024-09-14T03:32:59.748 INFO:teuthology.task.kernel:Checking kernel version of mon.a, want "5.14.0-505.el9.x86_64"... 2024-09-14T03:32:59.749 DEBUG:teuthology.orchestra.run.smithi083:> uname -r 2024-09-14T03:32:59.763 INFO:teuthology.orchestra.run.smithi083.stdout:5.14.0-505.el9.x86_64 2024-09-14T03:32:59.763 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-505.el9.x86_64 vs 5.14.0-505.el9.x86_64 2024-09-14T03:32:59.764 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-09-14T03:32:59.764 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-09-14T03:33:00.764 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2024-09-14T03:33:00.765 DEBUG:teuthology.orchestra.run.smithi083:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-09-14T03:33:00.858 INFO:teuthology.orchestra.run.smithi083.stdout:ttyS1 2024-09-14T03:33:00.879 DEBUG:teuthology.parallel:result is None 2024-09-14T03:33:00.879 INFO:teuthology.run_tasks:Running task internal.base... 2024-09-14T03:33:00.886 INFO:teuthology.task.internal:Creating test directory... 2024-09-14T03:33:00.887 DEBUG:teuthology.orchestra.run.smithi083:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-09-14T03:33:00.902 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-09-14T03:33:00.907 INFO:teuthology.run_tasks:Running task internal.archive... 2024-09-14T03:33:00.941 INFO:teuthology.task.internal:Creating archive directory... 2024-09-14T03:33:00.941 DEBUG:teuthology.orchestra.run.smithi083:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-09-14T03:33:00.971 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-09-14T03:33:00.977 INFO:teuthology.task.internal:Enabling coredump saving... 2024-09-14T03:33:00.978 DEBUG:teuthology.orchestra.run.smithi083:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-09-14T03:33:01.041 INFO:teuthology.orchestra.run.smithi083.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-14T03:33:01.051 INFO:teuthology.orchestra.run.smithi083.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-14T03:33:01.053 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-09-14T03:33:01.060 INFO:teuthology.task.internal:Configuring sudo... 2024-09-14T03:33:01.061 DEBUG:teuthology.orchestra.run.smithi083:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-09-14T03:33:01.119 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-09-14T03:33:01.127 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-09-14T03:33:01.128 DEBUG:teuthology.orchestra.run.smithi083:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-09-14T03:33:01.174 DEBUG:teuthology.orchestra.run.smithi083:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-14T03:33:01.258 DEBUG:teuthology.orchestra.run.smithi083:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-14T03:33:01.308 DEBUG:teuthology.orchestra.run.smithi083:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-14T03:33:01.387 DEBUG:teuthology.orchestra.run.smithi083:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-14T03:33:01.431 DEBUG:teuthology.orchestra.run.smithi083:> set -ex 2024-09-14T03:33:01.431 DEBUG:teuthology.orchestra.run.smithi083:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-09-14T03:33:01.496 DEBUG:teuthology.orchestra.run.smithi083:> sudo service rsyslog restart 2024-09-14T03:33:01.589 INFO:teuthology.orchestra.run.smithi083.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-14T03:33:01.845 INFO:teuthology.run_tasks:Running task internal.timer... 2024-09-14T03:33:01.856 INFO:teuthology.task.internal:Starting timer... 2024-09-14T03:33:01.856 INFO:teuthology.run_tasks:Running task pcp... 2024-09-14T03:33:01.900 INFO:teuthology.run_tasks:Running task selinux... 2024-09-14T03:33:01.993 DEBUG:teuthology.task:Applying overrides for task selinux: {'allowlist': ['scontext=system_u:system_r:logrotate_t:s0']} 2024-09-14T03:33:01.993 DEBUG:teuthology.orchestra.run.smithi083:> sudo service auditd rotate 2024-09-14T03:33:02.055 INFO:teuthology.orchestra.run.smithi083.stdout:Rotating logs: 2024-09-14T03:33:02.057 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-09-14T03:33:02.058 DEBUG:teuthology.orchestra.run.smithi083:> /usr/sbin/getenforce 2024-09-14T03:33:02.079 INFO:teuthology.orchestra.run.smithi083.stdout:Permissive 2024-09-14T03:33:02.079 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi083.front.sepia.ceph.com': 'permissive'} 2024-09-14T03:33:02.080 DEBUG:teuthology.orchestra.run.smithi083:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-09-14T03:33:02.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T03:33:02.144 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-09-14T03:33:02.145 DEBUG:teuthology.orchestra.run.smithi083:> sudo /usr/sbin/setenforce permissive 2024-09-14T03:33:02.212 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-09-14T03:33:02.226 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-09-14T03:33:02.239 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-09-14T03:33:02.240 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi083.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-09-14T03:36:42.561 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi083.front.sepia.ceph.com')] 2024-09-14T03:36:42.562 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi083.front.sepia.ceph.com' 2024-09-14T03:36:42.563 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi083.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-14T03:36:42.638 DEBUG:teuthology.orchestra.run.smithi083:> true 2024-09-14T03:36:42.723 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi083.front.sepia.ceph.com' 2024-09-14T03:36:42.724 INFO:teuthology.run_tasks:Running task clock... 2024-09-14T03:36:42.733 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-09-14T03:36:42.734 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-09-14T03:36:42.734 DEBUG:teuthology.orchestra.run.smithi083:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-14T03:36:42.792 INFO:teuthology.orchestra.run.smithi083.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-09-14T03:36:42.806 INFO:teuthology.orchestra.run.smithi083.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-09-14T03:36:42.830 INFO:teuthology.orchestra.run.smithi083.stderr:sudo: ntpd: command not found 2024-09-14T03:36:42.841 INFO:teuthology.orchestra.run.smithi083.stdout:506 Cannot talk to daemon 2024-09-14T03:36:42.856 INFO:teuthology.orchestra.run.smithi083.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-09-14T03:36:42.870 INFO:teuthology.orchestra.run.smithi083.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-09-14T03:36:42.927 INFO:teuthology.orchestra.run.smithi083.stderr:bash: line 1: ntpq: command not found 2024-09-14T03:36:42.930 INFO:teuthology.orchestra.run.smithi083.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-14T03:36:42.930 INFO:teuthology.orchestra.run.smithi083.stdout:=============================================================================== 2024-09-14T03:36:42.930 INFO:teuthology.orchestra.run.smithi083.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-14T03:36:42.930 INFO:teuthology.orchestra.run.smithi083.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-14T03:36:42.930 INFO:teuthology.orchestra.run.smithi083.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-14T03:36:42.930 INFO:teuthology.orchestra.run.smithi083.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-14T03:36:42.931 INFO:teuthology.run_tasks:Running task install... 2024-09-14T03:36:42.939 DEBUG:teuthology.task.install:project ceph 2024-09-14T03:36:42.940 DEBUG:teuthology.task.install:INSTALL overrides: {'ceph': {'flavor': 'default', 'sha1': 'a2782315255a3e7d5f6cdc9eb35785187ff6ddf8'}} 2024-09-14T03:36:42.940 DEBUG:teuthology.task.install:config {'branch': 'squid', 'exclude_packages': ['librados3', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'cephadm', 'ceph-volume'], 'extra_packages': ['librados2'], 'flavor': 'default', 'sha1': 'a2782315255a3e7d5f6cdc9eb35785187ff6ddf8'} 2024-09-14T03:36:42.940 INFO:teuthology.task.install:Using flavor: default 2024-09-14T03:36:42.945 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-fuse', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2024-09-14T03:36:42.945 INFO:teuthology.task.install:extra packages: ['librados2'] 2024-09-14T03:36:42.946 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': 'squid', 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': ['librados3', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'cephadm', 'ceph-volume'], 'extra_packages': ['librados2'], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': 'a2782315255a3e7d5f6cdc9eb35785187ff6ddf8', 'tag': None, 'wait_for_package': False} 2024-09-14T03:36:42.946 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-14T03:36:42.946 INFO:teuthology.packaging:ref: None 2024-09-14T03:36:42.946 INFO:teuthology.packaging:tag: None 2024-09-14T03:36:42.946 INFO:teuthology.packaging:branch: squid 2024-09-14T03:36:42.946 INFO:teuthology.packaging:sha1: a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T03:36:42.946 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=squid 2024-09-14T03:36:43.185 INFO:teuthology.task.install.rpm:Pulling from https://1.chacra.ceph.com/r/ceph/squid/ecab7be58963a5bd03910cd73eef66405d7fdd50/centos/9/flavors/default/ 2024-09-14T03:36:43.186 INFO:teuthology.task.install.rpm:Package version is 19.1.1-324.gecab7be5 2024-09-14T03:36:43.300 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://1.chacra.ceph.com/r/ceph/squid/ecab7be58963a5bd03910cd73eef66405d7fdd50/centos/9/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://1.chacra.ceph.com/r/ceph/squid/ecab7be58963a5bd03910cd73eef66405d7fdd50/centos/9/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://1.chacra.ceph.com/r/ceph/squid/ecab7be58963a5bd03910cd73eef66405d7fdd50/centos/9/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2024-09-14T03:36:43.301 DEBUG:teuthology.orchestra.run.smithi083:> set -ex 2024-09-14T03:36:43.301 DEBUG:teuthology.orchestra.run.smithi083:> sudo dd of=/etc/yum.repos.d/ceph.repo 2024-09-14T03:36:43.327 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, ceph-immutable-object-cache, ceph-mgr, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd, librados2 on remote rpm x86_64 2024-09-14T03:36:43.327 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-14T03:36:43.327 INFO:teuthology.packaging:ref: None 2024-09-14T03:36:43.328 INFO:teuthology.packaging:tag: None 2024-09-14T03:36:43.328 INFO:teuthology.packaging:branch: squid 2024-09-14T03:36:43.328 INFO:teuthology.packaging:sha1: a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T03:36:43.328 DEBUG:teuthology.orchestra.run.smithi083:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;ref/squid/;g' /etc/yum.repos.d/ceph.repo ; fi 2024-09-14T03:36:43.393 DEBUG:teuthology.orchestra.run.smithi083:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2024-09-14T03:36:43.468 DEBUG:teuthology.orchestra.run.smithi083:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2024-09-14T03:36:43.532 INFO:teuthology.orchestra.run.smithi083.stdout:check_obsoletes = 1 2024-09-14T03:36:43.534 DEBUG:teuthology.orchestra.run.smithi083:> sudo yum clean all 2024-09-14T03:36:43.759 INFO:teuthology.orchestra.run.smithi083.stdout:45 files removed 2024-09-14T03:36:43.786 DEBUG:teuthology.orchestra.run.smithi083:> sudo yum -y install ceph-radosgw 2024-09-14T03:36:44.354 INFO:teuthology.orchestra.run.smithi083.stdout:ceph packages for x86_64 293 kB/s | 84 kB 00:00 2024-09-14T03:36:44.638 INFO:teuthology.orchestra.run.smithi083.stdout:ceph noarch packages 49 kB/s | 12 kB 00:00 2024-09-14T03:36:44.914 INFO:teuthology.orchestra.run.smithi083.stdout:ceph source packages 7.9 kB/s | 1.9 kB 00:00 2024-09-14T03:36:46.630 INFO:teuthology.orchestra.run.smithi083.stdout:CentOS Stream 9 - BaseOS 4.9 MB/s | 8.2 MB 00:01 2024-09-14T03:36:51.874 INFO:teuthology.orchestra.run.smithi083.stdout:CentOS Stream 9 - AppStream 4.9 MB/s | 20 MB 00:04 2024-09-14T03:36:56.391 INFO:teuthology.orchestra.run.smithi083.stdout:CentOS Stream 9 - CRB 9.2 MB/s | 6.5 MB 00:00 2024-09-14T03:36:58.041 INFO:teuthology.orchestra.run.smithi083.stdout:CentOS Stream 9 - Extras packages 43 kB/s | 19 kB 00:00 2024-09-14T03:36:59.287 INFO:teuthology.orchestra.run.smithi083.stdout:Extra Packages for Enterprise Linux 20 MB/s | 23 MB 00:01 2024-09-14T03:37:04.777 INFO:teuthology.orchestra.run.smithi083.stdout:lab-extras 31 kB/s | 1.7 kB 00:00 2024-09-14T03:37:07.153 INFO:teuthology.orchestra.run.smithi083.stdout:Dependencies resolved. 2024-09-14T03:37:07.155 INFO:teuthology.orchestra.run.smithi083.stdout:================================================================================ 2024-09-14T03:37:07.155 INFO:teuthology.orchestra.run.smithi083.stdout: Package Arch Version Repository Size 2024-09-14T03:37:07.155 INFO:teuthology.orchestra.run.smithi083.stdout:================================================================================ 2024-09-14T03:37:07.155 INFO:teuthology.orchestra.run.smithi083.stdout:Installing: 2024-09-14T03:37:07.156 INFO:teuthology.orchestra.run.smithi083.stdout: ceph-radosgw x86_64 2:19.1.1-324.gecab7be5.el9 ceph 11 M 2024-09-14T03:37:07.156 INFO:teuthology.orchestra.run.smithi083.stdout:Upgrading: 2024-09-14T03:37:07.156 INFO:teuthology.orchestra.run.smithi083.stdout: librados2 x86_64 2:19.1.1-324.gecab7be5.el9 ceph 3.4 M 2024-09-14T03:37:07.156 INFO:teuthology.orchestra.run.smithi083.stdout: librbd1 x86_64 2:19.1.1-324.gecab7be5.el9 ceph 3.2 M 2024-09-14T03:37:07.156 INFO:teuthology.orchestra.run.smithi083.stdout:Installing dependencies: 2024-09-14T03:37:07.156 INFO:teuthology.orchestra.run.smithi083.stdout: boost-program-options x86_64 1.75.0-8.el9 appstream 107 k 2024-09-14T03:37:07.156 INFO:teuthology.orchestra.run.smithi083.stdout: ceph-base x86_64 2:19.1.1-324.gecab7be5.el9 ceph 5.4 M 2024-09-14T03:37:07.156 INFO:teuthology.orchestra.run.smithi083.stdout: ceph-common x86_64 2:19.1.1-324.gecab7be5.el9 ceph 21 M 2024-09-14T03:37:07.156 INFO:teuthology.orchestra.run.smithi083.stdout: ceph-selinux x86_64 2:19.1.1-324.gecab7be5.el9 ceph 25 k 2024-09-14T03:37:07.156 INFO:teuthology.orchestra.run.smithi083.stdout: gperftools-libs x86_64 2.9.1-3.el9 epel 308 k 2024-09-14T03:37:07.156 INFO:teuthology.orchestra.run.smithi083.stdout: libarrow x86_64 9.0.0-11.el9 epel 4.4 M 2024-09-14T03:37:07.156 INFO:teuthology.orchestra.run.smithi083.stdout: libarrow-doc noarch 9.0.0-11.el9 epel 26 k 2024-09-14T03:37:07.156 INFO:teuthology.orchestra.run.smithi083.stdout: libcephfs2 x86_64 2:19.1.1-324.gecab7be5.el9 ceph 752 k 2024-09-14T03:37:07.157 INFO:teuthology.orchestra.run.smithi083.stdout: liboath x86_64 2.6.7-2.el9 epel 49 k 2024-09-14T03:37:07.157 INFO:teuthology.orchestra.run.smithi083.stdout: libpmemobj x86_64 1.12.1-1.el9 appstream 160 k 2024-09-14T03:37:07.157 INFO:teuthology.orchestra.run.smithi083.stdout: librabbitmq x86_64 0.11.0-7.el9 appstream 45 k 2024-09-14T03:37:07.157 INFO:teuthology.orchestra.run.smithi083.stdout: libradosstriper1 x86_64 2:19.1.1-324.gecab7be5.el9 ceph 507 k 2024-09-14T03:37:07.157 INFO:teuthology.orchestra.run.smithi083.stdout: librdkafka x86_64 1.6.1-102.el9 appstream 662 k 2024-09-14T03:37:07.157 INFO:teuthology.orchestra.run.smithi083.stdout: librgw2 x86_64 2:19.1.1-324.gecab7be5.el9 ceph 5.3 M 2024-09-14T03:37:07.157 INFO:teuthology.orchestra.run.smithi083.stdout: libunwind x86_64 1.6.2-1.el9 epel 67 k 2024-09-14T03:37:07.157 INFO:teuthology.orchestra.run.smithi083.stdout: lttng-ust x86_64 2.12.0-6.el9 appstream 292 k 2024-09-14T03:37:07.157 INFO:teuthology.orchestra.run.smithi083.stdout: parquet-libs x86_64 9.0.0-11.el9 epel 839 k 2024-09-14T03:37:07.157 INFO:teuthology.orchestra.run.smithi083.stdout: python3-ceph-argparse x86_64 2:19.1.1-324.gecab7be5.el9 ceph 45 k 2024-09-14T03:37:07.157 INFO:teuthology.orchestra.run.smithi083.stdout: python3-ceph-common x86_64 2:19.1.1-324.gecab7be5.el9 ceph 136 k 2024-09-14T03:37:07.157 INFO:teuthology.orchestra.run.smithi083.stdout: python3-cephfs x86_64 2:19.1.1-324.gecab7be5.el9 ceph 162 k 2024-09-14T03:37:07.157 INFO:teuthology.orchestra.run.smithi083.stdout: python3-prettytable noarch 0.7.2-27.el9 appstream 42 k 2024-09-14T03:37:07.158 INFO:teuthology.orchestra.run.smithi083.stdout: python3-rados x86_64 2:19.1.1-324.gecab7be5.el9 ceph 321 k 2024-09-14T03:37:07.158 INFO:teuthology.orchestra.run.smithi083.stdout: python3-rbd x86_64 2:19.1.1-324.gecab7be5.el9 ceph 301 k 2024-09-14T03:37:07.158 INFO:teuthology.orchestra.run.smithi083.stdout: python3-rgw x86_64 2:19.1.1-324.gecab7be5.el9 ceph 99 k 2024-09-14T03:37:07.158 INFO:teuthology.orchestra.run.smithi083.stdout: qatlib x86_64 24.02.0-1.el9 appstream 221 k 2024-09-14T03:37:07.158 INFO:teuthology.orchestra.run.smithi083.stdout: qatzip-libs x86_64 1.2.0-1.el9 appstream 47 k 2024-09-14T03:37:07.158 INFO:teuthology.orchestra.run.smithi083.stdout: re2 x86_64 1:20211101-20.el9 epel 191 k 2024-09-14T03:37:07.158 INFO:teuthology.orchestra.run.smithi083.stdout: thrift x86_64 0.15.0-2.el9 epel 1.6 M 2024-09-14T03:37:07.158 INFO:teuthology.orchestra.run.smithi083.stdout:Installing weak dependencies: 2024-09-14T03:37:07.158 INFO:teuthology.orchestra.run.smithi083.stdout: qatlib-service x86_64 24.02.0-1.el9 appstream 36 k 2024-09-14T03:37:07.158 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:37:07.158 INFO:teuthology.orchestra.run.smithi083.stdout:Transaction Summary 2024-09-14T03:37:07.158 INFO:teuthology.orchestra.run.smithi083.stdout:================================================================================ 2024-09-14T03:37:07.158 INFO:teuthology.orchestra.run.smithi083.stdout:Install 30 Packages 2024-09-14T03:37:07.158 INFO:teuthology.orchestra.run.smithi083.stdout:Upgrade 2 Packages 2024-09-14T03:37:07.159 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:37:07.159 INFO:teuthology.orchestra.run.smithi083.stdout:Total download size: 60 M 2024-09-14T03:37:07.159 INFO:teuthology.orchestra.run.smithi083.stdout:Downloading Packages: 2024-09-14T03:37:08.136 INFO:teuthology.orchestra.run.smithi083.stdout:(1/32): ceph-base-19.1.1-324.gecab7be5.el9.x86_ 8.5 MB/s | 5.4 MB 00:00 2024-09-14T03:37:08.186 INFO:teuthology.orchestra.run.smithi083.stdout:(2/32): ceph-selinux-19.1.1-324.gecab7be5.el9.x 503 kB/s | 25 kB 00:00 2024-09-14T03:37:08.287 INFO:teuthology.orchestra.run.smithi083.stdout:(3/32): libcephfs2-19.1.1-324.gecab7be5.el9.x86 7.3 MB/s | 752 kB 00:00 2024-09-14T03:37:08.379 INFO:teuthology.orchestra.run.smithi083.stdout:(4/32): libradosstriper1-19.1.1-324.gecab7be5.e 5.4 MB/s | 507 kB 00:00 2024-09-14T03:37:08.487 INFO:teuthology.orchestra.run.smithi083.stdout:(5/32): ceph-radosgw-19.1.1-324.gecab7be5.el9.x 11 MB/s | 11 MB 00:00 2024-09-14T03:37:08.538 INFO:teuthology.orchestra.run.smithi083.stdout:(6/32): python3-ceph-argparse-19.1.1-324.gecab7 906 kB/s | 45 kB 00:00 2024-09-14T03:37:08.580 INFO:teuthology.orchestra.run.smithi083.stdout:(7/32): python3-ceph-common-19.1.1-324.gecab7be 3.2 MB/s | 136 kB 00:00 2024-09-14T03:37:08.630 INFO:teuthology.orchestra.run.smithi083.stdout:(8/32): python3-cephfs-19.1.1-324.gecab7be5.el9 3.2 MB/s | 162 kB 00:00 2024-09-14T03:37:08.697 INFO:teuthology.orchestra.run.smithi083.stdout:(9/32): python3-rados-19.1.1-324.gecab7be5.el9. 4.7 MB/s | 321 kB 00:00 2024-09-14T03:37:08.922 INFO:teuthology.orchestra.run.smithi083.stdout:(10/32): ceph-common-19.1.1-324.gecab7be5.el9.x 15 MB/s | 21 MB 00:01 2024-09-14T03:37:08.997 INFO:teuthology.orchestra.run.smithi083.stdout:(11/32): librgw2-19.1.1-324.gecab7be5.el9.x86_6 8.6 MB/s | 5.3 MB 00:00 2024-09-14T03:37:09.023 INFO:teuthology.orchestra.run.smithi083.stdout:(12/32): python3-rbd-19.1.1-324.gecab7be5.el9.x 924 kB/s | 301 kB 00:00 2024-09-14T03:37:09.048 INFO:teuthology.orchestra.run.smithi083.stdout:(13/32): python3-rgw-19.1.1-324.gecab7be5.el9.x 792 kB/s | 99 kB 00:00 2024-09-14T03:37:09.131 INFO:teuthology.orchestra.run.smithi083.stdout:(14/32): boost-program-options-1.75.0-8.el9.x86 800 kB/s | 107 kB 00:00 2024-09-14T03:37:09.157 INFO:teuthology.orchestra.run.smithi083.stdout:(15/32): librabbitmq-0.11.0-7.el9.x86_64.rpm 417 kB/s | 45 kB 00:00 2024-09-14T03:37:09.182 INFO:teuthology.orchestra.run.smithi083.stdout:(16/32): libpmemobj-1.12.1-1.el9.x86_64.rpm 1.0 MB/s | 160 kB 00:00 2024-09-14T03:37:09.224 INFO:teuthology.orchestra.run.smithi083.stdout:(17/32): python3-prettytable-0.7.2-27.el9.noarc 1.0 MB/s | 42 kB 00:00 2024-09-14T03:37:09.249 INFO:teuthology.orchestra.run.smithi083.stdout:(18/32): librdkafka-1.6.1-102.el9.x86_64.rpm 5.5 MB/s | 662 kB 00:00 2024-09-14T03:37:09.274 INFO:teuthology.orchestra.run.smithi083.stdout:(19/32): lttng-ust-2.12.0-6.el9.x86_64.rpm 2.4 MB/s | 292 kB 00:00 2024-09-14T03:37:09.308 INFO:teuthology.orchestra.run.smithi083.stdout:(20/32): qatlib-service-24.02.0-1.el9.x86_64.rp 612 kB/s | 36 kB 00:00 2024-09-14T03:37:09.333 INFO:teuthology.orchestra.run.smithi083.stdout:(21/32): qatzip-libs-1.2.0-1.el9.x86_64.rpm 810 kB/s | 47 kB 00:00 2024-09-14T03:37:09.359 INFO:teuthology.orchestra.run.smithi083.stdout:(22/32): qatlib-24.02.0-1.el9.x86_64.rpm 1.6 MB/s | 221 kB 00:00 2024-09-14T03:37:09.418 INFO:teuthology.orchestra.run.smithi083.stdout:(23/32): gperftools-libs-2.9.1-3.el9.x86_64.rpm 2.8 MB/s | 308 kB 00:00 2024-09-14T03:37:09.442 INFO:teuthology.orchestra.run.smithi083.stdout:(24/32): liboath-2.6.7-2.el9.x86_64.rpm 2.1 MB/s | 49 kB 00:00 2024-09-14T03:37:09.468 INFO:teuthology.orchestra.run.smithi083.stdout:(25/32): libarrow-doc-9.0.0-11.el9.noarch.rpm 275 kB/s | 26 kB 00:00 2024-09-14T03:37:09.493 INFO:teuthology.orchestra.run.smithi083.stdout:(26/32): libunwind-1.6.2-1.el9.x86_64.rpm 1.3 MB/s | 67 kB 00:00 2024-09-14T03:37:09.527 INFO:teuthology.orchestra.run.smithi083.stdout:(27/32): re2-20211101-20.el9.x86_64.rpm 5.6 MB/s | 191 kB 00:00 2024-09-14T03:37:09.577 INFO:teuthology.orchestra.run.smithi083.stdout:(28/32): parquet-libs-9.0.0-11.el9.x86_64.rpm 7.5 MB/s | 839 kB 00:00 2024-09-14T03:37:09.652 INFO:teuthology.orchestra.run.smithi083.stdout:(29/32): thrift-0.15.0-2.el9.x86_64.rpm 13 MB/s | 1.6 MB 00:00 2024-09-14T03:37:09.761 INFO:teuthology.orchestra.run.smithi083.stdout:(30/32): libarrow-9.0.0-11.el9.x86_64.rpm 10 MB/s | 4.4 MB 00:00 2024-09-14T03:37:09.886 INFO:teuthology.orchestra.run.smithi083.stdout:(31/32): librados2-19.1.1-324.gecab7be5.el9.x86 11 MB/s | 3.4 MB 00:00 2024-09-14T03:37:09.936 INFO:teuthology.orchestra.run.smithi083.stdout:(32/32): librbd1-19.1.1-324.gecab7be5.el9.x86_6 11 MB/s | 3.2 MB 00:00 2024-09-14T03:37:09.941 INFO:teuthology.orchestra.run.smithi083.stdout:-------------------------------------------------------------------------------- 2024-09-14T03:37:09.942 INFO:teuthology.orchestra.run.smithi083.stdout:Total 22 MB/s | 60 MB 00:02 2024-09-14T03:37:10.045 INFO:teuthology.orchestra.run.smithi083.stdout:Running transaction check 2024-09-14T03:37:10.081 INFO:teuthology.orchestra.run.smithi083.stdout:Transaction check succeeded. 2024-09-14T03:37:10.081 INFO:teuthology.orchestra.run.smithi083.stdout:Running transaction test 2024-09-14T03:37:10.767 INFO:teuthology.orchestra.run.smithi083.stdout:Transaction test succeeded. 2024-09-14T03:37:10.768 INFO:teuthology.orchestra.run.smithi083.stdout:Running transaction 2024-09-14T03:37:12.239 INFO:teuthology.orchestra.run.smithi083.stdout: Preparing : 1/1 2024-09-14T03:37:12.327 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : lttng-ust-2.12.0-6.el9.x86_64 1/34 2024-09-14T03:37:12.367 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : librdkafka-1.6.1-102.el9.x86_64 2/34 2024-09-14T03:37:12.392 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : librabbitmq-0.11.0-7.el9.x86_64 3/34 2024-09-14T03:37:12.415 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : qatlib-service-24.02.0-1.el9.x86_64 4/34 2024-09-14T03:37:12.441 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: qatlib-service-24.02.0-1.el9.x86_64 4/34 2024-09-14T03:37:12.592 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: qatlib-24.02.0-1.el9.x86_64 5/34 2024-09-14T03:37:12.645 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : qatlib-24.02.0-1.el9.x86_64 5/34 2024-09-14T03:37:13.039 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : thrift-0.15.0-2.el9.x86_64 6/34 2024-09-14T03:37:13.094 INFO:teuthology.orchestra.run.smithi083.stdout: Upgrading : librados2-2:19.1.1-324.gecab7be5.el9.x86_64 7/34 2024-09-14T03:37:13.186 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: librados2-2:19.1.1-324.gecab7be5.el9.x86_64 7/34 2024-09-14T03:37:13.276 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : python3-rados-2:19.1.1-324.gecab7be5.el9.x86_64 8/34 2024-09-14T03:37:13.312 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : libcephfs2-2:19.1.1-324.gecab7be5.el9.x86_64 9/34 2024-09-14T03:37:13.375 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: libcephfs2-2:19.1.1-324.gecab7be5.el9.x86_64 9/34 2024-09-14T03:37:13.412 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : liboath-2.6.7-2.el9.x86_64 10/34 2024-09-14T03:37:13.457 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : python3-ceph-argparse-2:19.1.1-324.gecab7be5.el9.x 11/34 2024-09-14T03:37:13.526 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : python3-cephfs-2:19.1.1-324.gecab7be5.el9.x86_64 12/34 2024-09-14T03:37:13.547 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : libradosstriper1-2:19.1.1-324.gecab7be5.el9.x86_64 13/34 2024-09-14T03:37:13.595 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: libradosstriper1-2:19.1.1-324.gecab7be5.el9.x86_64 13/34 2024-09-14T03:37:13.629 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : qatzip-libs-1.2.0-1.el9.x86_64 14/34 2024-09-14T03:37:13.739 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : re2-1:20211101-20.el9.x86_64 15/34 2024-09-14T03:37:13.785 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : libarrow-9.0.0-11.el9.x86_64 16/34 2024-09-14T03:37:13.821 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : libunwind-1.6.2-1.el9.x86_64 17/34 2024-09-14T03:37:13.862 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : gperftools-libs-2.9.1-3.el9.x86_64 18/34 2024-09-14T03:37:13.891 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : libarrow-doc-9.0.0-11.el9.noarch 19/34 2024-09-14T03:37:13.927 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : python3-prettytable-0.7.2-27.el9.noarch 20/34 2024-09-14T03:37:14.248 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : libpmemobj-1.12.1-1.el9.x86_64 21/34 2024-09-14T03:37:14.287 INFO:teuthology.orchestra.run.smithi083.stdout: Upgrading : librbd1-2:19.1.1-324.gecab7be5.el9.x86_64 22/34 2024-09-14T03:37:14.378 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: librbd1-2:19.1.1-324.gecab7be5.el9.x86_64 22/34 2024-09-14T03:37:14.409 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : python3-rbd-2:19.1.1-324.gecab7be5.el9.x86_64 23/34 2024-09-14T03:37:14.452 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : boost-program-options-1.75.0-8.el9.x86_64 24/34 2024-09-14T03:37:14.934 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : parquet-libs-9.0.0-11.el9.x86_64 25/34 2024-09-14T03:37:14.956 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : librgw2-2:19.1.1-324.gecab7be5.el9.x86_64 26/34 2024-09-14T03:37:15.011 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: librgw2-2:19.1.1-324.gecab7be5.el9.x86_64 26/34 2024-09-14T03:37:15.088 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : python3-rgw-2:19.1.1-324.gecab7be5.el9.x86_64 27/34 2024-09-14T03:37:15.117 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : python3-ceph-common-2:19.1.1-324.gecab7be5.el9.x86 28/34 2024-09-14T03:37:17.383 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: ceph-common-2:19.1.1-324.gecab7be5.el9.x86_64 29/34 2024-09-14T03:37:17.422 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : ceph-common-2:19.1.1-324.gecab7be5.el9.x86_64 29/34 2024-09-14T03:37:17.986 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: ceph-common-2:19.1.1-324.gecab7be5.el9.x86_64 29/34 2024-09-14T03:37:18.025 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : ceph-base-2:19.1.1-324.gecab7be5.el9.x86_64 30/34 2024-09-14T03:37:18.116 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: ceph-base-2:19.1.1-324.gecab7be5.el9.x86_64 30/34 2024-09-14T03:37:18.116 INFO:teuthology.orchestra.run.smithi083.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /usr/lib/systemd/system/ceph.target. 2024-09-14T03:37:18.116 INFO:teuthology.orchestra.run.smithi083.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /usr/lib/systemd/system/ceph-crash.service. 2024-09-14T03:37:18.116 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:37:18.151 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : ceph-selinux-2:19.1.1-324.gecab7be5.el9.x86_64 31/34 2024-09-14T03:37:27.986 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: ceph-selinux-2:19.1.1-324.gecab7be5.el9.x86_64 31/34 2024-09-14T03:37:27.986 INFO:teuthology.orchestra.run.smithi083.stdout:skipping the directory /sys 2024-09-14T03:37:27.986 INFO:teuthology.orchestra.run.smithi083.stdout:skipping the directory /proc 2024-09-14T03:37:27.986 INFO:teuthology.orchestra.run.smithi083.stdout:skipping the directory /mnt 2024-09-14T03:37:27.986 INFO:teuthology.orchestra.run.smithi083.stdout:skipping the directory /var/tmp 2024-09-14T03:37:27.986 INFO:teuthology.orchestra.run.smithi083.stdout:skipping the directory /home 2024-09-14T03:37:27.986 INFO:teuthology.orchestra.run.smithi083.stdout:skipping the directory /root 2024-09-14T03:37:27.987 INFO:teuthology.orchestra.run.smithi083.stdout:skipping the directory /tmp 2024-09-14T03:37:27.987 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:37:28.956 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : ceph-radosgw-2:19.1.1-324.gecab7be5.el9.x86_64 32/34 2024-09-14T03:37:28.982 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: ceph-radosgw-2:19.1.1-324.gecab7be5.el9.x86_64 32/34 2024-09-14T03:37:28.982 INFO:teuthology.orchestra.run.smithi083.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-14T03:37:28.982 INFO:teuthology.orchestra.run.smithi083.stdout:Invalid unit name "ceph-radosgw@*.service" escaped as "ceph-radosgw@\x2a.service". 2024-09-14T03:37:28.982 INFO:teuthology.orchestra.run.smithi083.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-09-14T03:37:28.982 INFO:teuthology.orchestra.run.smithi083.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-09-14T03:37:28.982 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:37:28.984 INFO:teuthology.orchestra.run.smithi083.stdout: Cleanup : librbd1-2:16.2.4-5.el9.x86_64 33/34 2024-09-14T03:37:29.066 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: librbd1-2:16.2.4-5.el9.x86_64 33/34 2024-09-14T03:37:29.066 INFO:teuthology.orchestra.run.smithi083.stdout: Cleanup : librados2-2:16.2.4-5.el9.x86_64 34/34 2024-09-14T03:37:30.298 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: librados2-2:16.2.4-5.el9.x86_64 34/34 2024-09-14T03:37:30.298 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : ceph-base-2:19.1.1-324.gecab7be5.el9.x86_64 1/34 2024-09-14T03:37:30.298 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : ceph-common-2:19.1.1-324.gecab7be5.el9.x86_64 2/34 2024-09-14T03:37:30.298 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : ceph-radosgw-2:19.1.1-324.gecab7be5.el9.x86_64 3/34 2024-09-14T03:37:30.298 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : ceph-selinux-2:19.1.1-324.gecab7be5.el9.x86_64 4/34 2024-09-14T03:37:30.298 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : libcephfs2-2:19.1.1-324.gecab7be5.el9.x86_64 5/34 2024-09-14T03:37:30.298 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : libradosstriper1-2:19.1.1-324.gecab7be5.el9.x86_64 6/34 2024-09-14T03:37:30.299 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : librgw2-2:19.1.1-324.gecab7be5.el9.x86_64 7/34 2024-09-14T03:37:30.299 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : python3-ceph-argparse-2:19.1.1-324.gecab7be5.el9.x 8/34 2024-09-14T03:37:30.299 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : python3-ceph-common-2:19.1.1-324.gecab7be5.el9.x86 9/34 2024-09-14T03:37:30.299 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : python3-cephfs-2:19.1.1-324.gecab7be5.el9.x86_64 10/34 2024-09-14T03:37:30.299 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : python3-rados-2:19.1.1-324.gecab7be5.el9.x86_64 11/34 2024-09-14T03:37:30.299 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : python3-rbd-2:19.1.1-324.gecab7be5.el9.x86_64 12/34 2024-09-14T03:37:30.299 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : python3-rgw-2:19.1.1-324.gecab7be5.el9.x86_64 13/34 2024-09-14T03:37:30.299 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : boost-program-options-1.75.0-8.el9.x86_64 14/34 2024-09-14T03:37:30.299 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : libpmemobj-1.12.1-1.el9.x86_64 15/34 2024-09-14T03:37:30.299 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : librabbitmq-0.11.0-7.el9.x86_64 16/34 2024-09-14T03:37:30.299 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : librdkafka-1.6.1-102.el9.x86_64 17/34 2024-09-14T03:37:30.299 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : lttng-ust-2.12.0-6.el9.x86_64 18/34 2024-09-14T03:37:30.299 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : python3-prettytable-0.7.2-27.el9.noarch 19/34 2024-09-14T03:37:30.300 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : qatlib-24.02.0-1.el9.x86_64 20/34 2024-09-14T03:37:30.300 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : qatlib-service-24.02.0-1.el9.x86_64 21/34 2024-09-14T03:37:30.300 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : qatzip-libs-1.2.0-1.el9.x86_64 22/34 2024-09-14T03:37:30.300 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : gperftools-libs-2.9.1-3.el9.x86_64 23/34 2024-09-14T03:37:30.300 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : libarrow-9.0.0-11.el9.x86_64 24/34 2024-09-14T03:37:30.300 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : libarrow-doc-9.0.0-11.el9.noarch 25/34 2024-09-14T03:37:30.300 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : liboath-2.6.7-2.el9.x86_64 26/34 2024-09-14T03:37:30.300 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : libunwind-1.6.2-1.el9.x86_64 27/34 2024-09-14T03:37:30.300 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : parquet-libs-9.0.0-11.el9.x86_64 28/34 2024-09-14T03:37:30.301 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : re2-1:20211101-20.el9.x86_64 29/34 2024-09-14T03:37:30.301 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : thrift-0.15.0-2.el9.x86_64 30/34 2024-09-14T03:37:30.301 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : librados2-2:19.1.1-324.gecab7be5.el9.x86_64 31/34 2024-09-14T03:37:30.301 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : librados2-2:16.2.4-5.el9.x86_64 32/34 2024-09-14T03:37:30.301 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : librbd1-2:19.1.1-324.gecab7be5.el9.x86_64 33/34 2024-09-14T03:37:31.375 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : librbd1-2:16.2.4-5.el9.x86_64 34/34 2024-09-14T03:37:31.375 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:37:31.375 INFO:teuthology.orchestra.run.smithi083.stdout:Upgraded: 2024-09-14T03:37:31.376 INFO:teuthology.orchestra.run.smithi083.stdout: librados2-2:19.1.1-324.gecab7be5.el9.x86_64 2024-09-14T03:37:31.376 INFO:teuthology.orchestra.run.smithi083.stdout: librbd1-2:19.1.1-324.gecab7be5.el9.x86_64 2024-09-14T03:37:31.376 INFO:teuthology.orchestra.run.smithi083.stdout:Installed: 2024-09-14T03:37:31.376 INFO:teuthology.orchestra.run.smithi083.stdout: boost-program-options-1.75.0-8.el9.x86_64 2024-09-14T03:37:31.376 INFO:teuthology.orchestra.run.smithi083.stdout: ceph-base-2:19.1.1-324.gecab7be5.el9.x86_64 2024-09-14T03:37:31.376 INFO:teuthology.orchestra.run.smithi083.stdout: ceph-common-2:19.1.1-324.gecab7be5.el9.x86_64 2024-09-14T03:37:31.376 INFO:teuthology.orchestra.run.smithi083.stdout: ceph-radosgw-2:19.1.1-324.gecab7be5.el9.x86_64 2024-09-14T03:37:31.376 INFO:teuthology.orchestra.run.smithi083.stdout: ceph-selinux-2:19.1.1-324.gecab7be5.el9.x86_64 2024-09-14T03:37:31.376 INFO:teuthology.orchestra.run.smithi083.stdout: gperftools-libs-2.9.1-3.el9.x86_64 2024-09-14T03:37:31.376 INFO:teuthology.orchestra.run.smithi083.stdout: libarrow-9.0.0-11.el9.x86_64 2024-09-14T03:37:31.376 INFO:teuthology.orchestra.run.smithi083.stdout: libarrow-doc-9.0.0-11.el9.noarch 2024-09-14T03:37:31.376 INFO:teuthology.orchestra.run.smithi083.stdout: libcephfs2-2:19.1.1-324.gecab7be5.el9.x86_64 2024-09-14T03:37:31.376 INFO:teuthology.orchestra.run.smithi083.stdout: liboath-2.6.7-2.el9.x86_64 2024-09-14T03:37:31.377 INFO:teuthology.orchestra.run.smithi083.stdout: libpmemobj-1.12.1-1.el9.x86_64 2024-09-14T03:37:31.377 INFO:teuthology.orchestra.run.smithi083.stdout: librabbitmq-0.11.0-7.el9.x86_64 2024-09-14T03:37:31.377 INFO:teuthology.orchestra.run.smithi083.stdout: libradosstriper1-2:19.1.1-324.gecab7be5.el9.x86_64 2024-09-14T03:37:31.377 INFO:teuthology.orchestra.run.smithi083.stdout: librdkafka-1.6.1-102.el9.x86_64 2024-09-14T03:37:31.377 INFO:teuthology.orchestra.run.smithi083.stdout: librgw2-2:19.1.1-324.gecab7be5.el9.x86_64 2024-09-14T03:37:31.377 INFO:teuthology.orchestra.run.smithi083.stdout: libunwind-1.6.2-1.el9.x86_64 2024-09-14T03:37:31.377 INFO:teuthology.orchestra.run.smithi083.stdout: lttng-ust-2.12.0-6.el9.x86_64 2024-09-14T03:37:31.377 INFO:teuthology.orchestra.run.smithi083.stdout: parquet-libs-9.0.0-11.el9.x86_64 2024-09-14T03:37:31.377 INFO:teuthology.orchestra.run.smithi083.stdout: python3-ceph-argparse-2:19.1.1-324.gecab7be5.el9.x86_64 2024-09-14T03:37:31.377 INFO:teuthology.orchestra.run.smithi083.stdout: python3-ceph-common-2:19.1.1-324.gecab7be5.el9.x86_64 2024-09-14T03:37:31.377 INFO:teuthology.orchestra.run.smithi083.stdout: python3-cephfs-2:19.1.1-324.gecab7be5.el9.x86_64 2024-09-14T03:37:31.377 INFO:teuthology.orchestra.run.smithi083.stdout: python3-prettytable-0.7.2-27.el9.noarch 2024-09-14T03:37:31.377 INFO:teuthology.orchestra.run.smithi083.stdout: python3-rados-2:19.1.1-324.gecab7be5.el9.x86_64 2024-09-14T03:37:31.377 INFO:teuthology.orchestra.run.smithi083.stdout: python3-rbd-2:19.1.1-324.gecab7be5.el9.x86_64 2024-09-14T03:37:31.378 INFO:teuthology.orchestra.run.smithi083.stdout: python3-rgw-2:19.1.1-324.gecab7be5.el9.x86_64 2024-09-14T03:37:31.378 INFO:teuthology.orchestra.run.smithi083.stdout: qatlib-24.02.0-1.el9.x86_64 2024-09-14T03:37:31.378 INFO:teuthology.orchestra.run.smithi083.stdout: qatlib-service-24.02.0-1.el9.x86_64 2024-09-14T03:37:31.378 INFO:teuthology.orchestra.run.smithi083.stdout: qatzip-libs-1.2.0-1.el9.x86_64 2024-09-14T03:37:31.378 INFO:teuthology.orchestra.run.smithi083.stdout: re2-1:20211101-20.el9.x86_64 2024-09-14T03:37:31.378 INFO:teuthology.orchestra.run.smithi083.stdout: thrift-0.15.0-2.el9.x86_64 2024-09-14T03:37:31.378 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:37:31.378 INFO:teuthology.orchestra.run.smithi083.stdout:Complete! 2024-09-14T03:37:31.749 DEBUG:teuthology.orchestra.run.smithi083:> sudo yum -y install ceph-test 2024-09-14T03:37:32.327 INFO:teuthology.orchestra.run.smithi083.stdout:Last metadata expiration check: 0:00:28 ago on Sat 14 Sep 2024 03:37:04 AM UTC. 2024-09-14T03:37:32.460 INFO:teuthology.orchestra.run.smithi083.stdout:Dependencies resolved. 2024-09-14T03:37:32.462 INFO:teuthology.orchestra.run.smithi083.stdout:================================================================================ 2024-09-14T03:37:32.462 INFO:teuthology.orchestra.run.smithi083.stdout: Package Arch Version Repository Size 2024-09-14T03:37:32.462 INFO:teuthology.orchestra.run.smithi083.stdout:================================================================================ 2024-09-14T03:37:32.462 INFO:teuthology.orchestra.run.smithi083.stdout:Installing: 2024-09-14T03:37:32.462 INFO:teuthology.orchestra.run.smithi083.stdout: ceph-test x86_64 2:19.1.1-324.gecab7be5.el9 ceph 49 M 2024-09-14T03:37:32.462 INFO:teuthology.orchestra.run.smithi083.stdout:Installing dependencies: 2024-09-14T03:37:32.463 INFO:teuthology.orchestra.run.smithi083.stdout: libcephsqlite x86_64 2:19.1.1-324.gecab7be5.el9 ceph 163 k 2024-09-14T03:37:32.463 INFO:teuthology.orchestra.run.smithi083.stdout: socat x86_64 1.7.4.1-6.el9 appstream 304 k 2024-09-14T03:37:32.463 INFO:teuthology.orchestra.run.smithi083.stdout: xmlstarlet x86_64 1.6.1-20.el9 appstream 64 k 2024-09-14T03:37:32.463 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:37:32.463 INFO:teuthology.orchestra.run.smithi083.stdout:Transaction Summary 2024-09-14T03:37:32.463 INFO:teuthology.orchestra.run.smithi083.stdout:================================================================================ 2024-09-14T03:37:32.464 INFO:teuthology.orchestra.run.smithi083.stdout:Install 4 Packages 2024-09-14T03:37:32.464 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:37:32.464 INFO:teuthology.orchestra.run.smithi083.stdout:Total download size: 49 M 2024-09-14T03:37:32.464 INFO:teuthology.orchestra.run.smithi083.stdout:Installed size: 209 M 2024-09-14T03:37:32.464 INFO:teuthology.orchestra.run.smithi083.stdout:Downloading Packages: 2024-09-14T03:37:32.738 INFO:teuthology.orchestra.run.smithi083.stdout:(1/4): socat-1.7.4.1-6.el9.x86_64.rpm 2.0 MB/s | 304 kB 00:00 2024-09-14T03:37:32.772 INFO:teuthology.orchestra.run.smithi083.stdout:(2/4): xmlstarlet-1.6.1-20.el9.x86_64.rpm 1.9 MB/s | 64 kB 00:00 2024-09-14T03:37:32.822 INFO:teuthology.orchestra.run.smithi083.stdout:(3/4): libcephsqlite-19.1.1-324.gecab7be5.el9.x 692 kB/s | 163 kB 00:00 2024-09-14T03:37:34.109 INFO:teuthology.orchestra.run.smithi083.stdout:(4/4): ceph-test-19.1.1-324.gecab7be5.el9.x86_6 32 MB/s | 49 MB 00:01 2024-09-14T03:37:34.109 INFO:teuthology.orchestra.run.smithi083.stdout:-------------------------------------------------------------------------------- 2024-09-14T03:37:34.110 INFO:teuthology.orchestra.run.smithi083.stdout:Total 30 MB/s | 49 MB 00:01 2024-09-14T03:37:34.137 INFO:teuthology.orchestra.run.smithi083.stdout:Running transaction check 2024-09-14T03:37:34.156 INFO:teuthology.orchestra.run.smithi083.stdout:Transaction check succeeded. 2024-09-14T03:37:34.156 INFO:teuthology.orchestra.run.smithi083.stdout:Running transaction test 2024-09-14T03:37:34.551 INFO:teuthology.orchestra.run.smithi083.stdout:Transaction test succeeded. 2024-09-14T03:37:34.552 INFO:teuthology.orchestra.run.smithi083.stdout:Running transaction 2024-09-14T03:37:35.085 INFO:teuthology.orchestra.run.smithi083.stdout: Preparing : 1/1 2024-09-14T03:37:35.164 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : xmlstarlet-1.6.1-20.el9.x86_64 1/4 2024-09-14T03:37:35.214 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : socat-1.7.4.1-6.el9.x86_64 2/4 2024-09-14T03:37:35.241 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : libcephsqlite-2:19.1.1-324.gecab7be5.el9.x86_64 3/4 2024-09-14T03:37:39.856 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: libcephsqlite-2:19.1.1-324.gecab7be5.el9.x86_64 3/4 2024-09-14T03:37:39.931 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : ceph-test-2:19.1.1-324.gecab7be5.el9.x86_64 4/4 2024-09-14T03:37:42.291 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: ceph-test-2:19.1.1-324.gecab7be5.el9.x86_64 4/4 2024-09-14T03:37:42.291 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : ceph-test-2:19.1.1-324.gecab7be5.el9.x86_64 1/4 2024-09-14T03:37:42.292 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : libcephsqlite-2:19.1.1-324.gecab7be5.el9.x86_64 2/4 2024-09-14T03:37:42.292 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : socat-1.7.4.1-6.el9.x86_64 3/4 2024-09-14T03:37:42.588 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : xmlstarlet-1.6.1-20.el9.x86_64 4/4 2024-09-14T03:37:42.588 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:37:42.588 INFO:teuthology.orchestra.run.smithi083.stdout:Installed: 2024-09-14T03:37:42.588 INFO:teuthology.orchestra.run.smithi083.stdout: ceph-test-2:19.1.1-324.gecab7be5.el9.x86_64 2024-09-14T03:37:42.588 INFO:teuthology.orchestra.run.smithi083.stdout: libcephsqlite-2:19.1.1-324.gecab7be5.el9.x86_64 2024-09-14T03:37:42.588 INFO:teuthology.orchestra.run.smithi083.stdout: socat-1.7.4.1-6.el9.x86_64 2024-09-14T03:37:42.588 INFO:teuthology.orchestra.run.smithi083.stdout: xmlstarlet-1.6.1-20.el9.x86_64 2024-09-14T03:37:42.588 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:37:42.589 INFO:teuthology.orchestra.run.smithi083.stdout:Complete! 2024-09-14T03:37:42.918 DEBUG:teuthology.orchestra.run.smithi083:> sudo yum -y install ceph 2024-09-14T03:37:43.503 INFO:teuthology.orchestra.run.smithi083.stdout:Last metadata expiration check: 0:00:39 ago on Sat 14 Sep 2024 03:37:04 AM UTC. 2024-09-14T03:37:43.656 INFO:teuthology.orchestra.run.smithi083.stdout:Dependencies resolved. 2024-09-14T03:37:43.658 INFO:teuthology.orchestra.run.smithi083.stdout:================================================================================ 2024-09-14T03:37:43.659 INFO:teuthology.orchestra.run.smithi083.stdout: Package Arch Version Repository Size 2024-09-14T03:37:43.659 INFO:teuthology.orchestra.run.smithi083.stdout:================================================================================ 2024-09-14T03:37:43.659 INFO:teuthology.orchestra.run.smithi083.stdout:Installing: 2024-09-14T03:37:43.659 INFO:teuthology.orchestra.run.smithi083.stdout: ceph x86_64 2:19.1.1-324.gecab7be5.el9 ceph 6.5 k 2024-09-14T03:37:43.659 INFO:teuthology.orchestra.run.smithi083.stdout:Installing dependencies: 2024-09-14T03:37:43.659 INFO:teuthology.orchestra.run.smithi083.stdout: ceph-mds x86_64 2:19.1.1-324.gecab7be5.el9 ceph 2.4 M 2024-09-14T03:37:43.659 INFO:teuthology.orchestra.run.smithi083.stdout: ceph-mgr x86_64 2:19.1.1-324.gecab7be5.el9 ceph 1.5 M 2024-09-14T03:37:43.659 INFO:teuthology.orchestra.run.smithi083.stdout: ceph-mgr-modules-core noarch 2:19.1.1-324.gecab7be5.el9 ceph-noarch 249 k 2024-09-14T03:37:43.659 INFO:teuthology.orchestra.run.smithi083.stdout: ceph-mon x86_64 2:19.1.1-324.gecab7be5.el9 ceph 4.7 M 2024-09-14T03:37:43.659 INFO:teuthology.orchestra.run.smithi083.stdout: ceph-osd x86_64 2:19.1.1-324.gecab7be5.el9 ceph 17 M 2024-09-14T03:37:43.659 INFO:teuthology.orchestra.run.smithi083.stdout: lua x86_64 5.4.4-4.el9 appstream 188 k 2024-09-14T03:37:43.659 INFO:teuthology.orchestra.run.smithi083.stdout: lua-devel x86_64 5.4.4-4.el9 crb 22 k 2024-09-14T03:37:43.659 INFO:teuthology.orchestra.run.smithi083.stdout: luarocks noarch 3.9.2-5.el9 epel 151 k 2024-09-14T03:37:43.660 INFO:teuthology.orchestra.run.smithi083.stdout: python3-bcrypt x86_64 3.2.2-1.el9 epel 43 k 2024-09-14T03:37:43.660 INFO:teuthology.orchestra.run.smithi083.stdout: python3-cffi x86_64 1.14.5-5.el9 baseos 253 k 2024-09-14T03:37:43.660 INFO:teuthology.orchestra.run.smithi083.stdout: python3-cheroot noarch 8.6.0-4.el9 epel 172 k 2024-09-14T03:37:43.660 INFO:teuthology.orchestra.run.smithi083.stdout: python3-cherrypy noarch 18.6.1-2.el9 epel 358 k 2024-09-14T03:37:43.660 INFO:teuthology.orchestra.run.smithi083.stdout: python3-cryptography x86_64 36.0.1-4.el9 baseos 1.2 M 2024-09-14T03:37:43.660 INFO:teuthology.orchestra.run.smithi083.stdout: python3-jaraco noarch 8.2.1-3.el9 epel 11 k 2024-09-14T03:37:43.660 INFO:teuthology.orchestra.run.smithi083.stdout: python3-jaraco-classes noarch 3.2.1-5.el9 epel 18 k 2024-09-14T03:37:43.660 INFO:teuthology.orchestra.run.smithi083.stdout: python3-jaraco-collections noarch 3.0.0-8.el9 epel 23 k 2024-09-14T03:37:43.660 INFO:teuthology.orchestra.run.smithi083.stdout: python3-jaraco-functools noarch 3.5.0-2.el9 epel 19 k 2024-09-14T03:37:43.660 INFO:teuthology.orchestra.run.smithi083.stdout: python3-jaraco-text noarch 3.2.0-6.el9 epel 20 k 2024-09-14T03:37:43.660 INFO:teuthology.orchestra.run.smithi083.stdout: python3-logutils noarch 0.3.5-21.el9 epel 46 k 2024-09-14T03:37:43.660 INFO:teuthology.orchestra.run.smithi083.stdout: python3-mako noarch 1.1.4-6.el9 appstream 172 k 2024-09-14T03:37:43.660 INFO:teuthology.orchestra.run.smithi083.stdout: python3-markupsafe x86_64 1.1.1-12.el9 appstream 35 k 2024-09-14T03:37:43.661 INFO:teuthology.orchestra.run.smithi083.stdout: python3-more-itertools noarch 8.12.0-2.el9 epel 79 k 2024-09-14T03:37:43.661 INFO:teuthology.orchestra.run.smithi083.stdout: python3-packaging noarch 20.9-5.el9 appstream 77 k 2024-09-14T03:37:43.661 INFO:teuthology.orchestra.run.smithi083.stdout: python3-pecan noarch 1.4.2-3.el9 epel 272 k 2024-09-14T03:37:43.661 INFO:teuthology.orchestra.run.smithi083.stdout: python3-ply noarch 3.11-14.el9 baseos 106 k 2024-09-14T03:37:43.661 INFO:teuthology.orchestra.run.smithi083.stdout: python3-portend noarch 3.1.0-2.el9 epel 16 k 2024-09-14T03:37:43.661 INFO:teuthology.orchestra.run.smithi083.stdout: python3-pyOpenSSL noarch 21.0.0-1.el9 epel 90 k 2024-09-14T03:37:43.661 INFO:teuthology.orchestra.run.smithi083.stdout: python3-pycparser noarch 2.20-6.el9 baseos 135 k 2024-09-14T03:37:43.661 INFO:teuthology.orchestra.run.smithi083.stdout: python3-pytz noarch 2021.1-5.el9 appstream 51 k 2024-09-14T03:37:43.661 INFO:teuthology.orchestra.run.smithi083.stdout: python3-requests noarch 2.25.1-8.el9 baseos 125 k 2024-09-14T03:37:43.661 INFO:teuthology.orchestra.run.smithi083.stdout: python3-tempora noarch 5.0.0-2.el9 epel 36 k 2024-09-14T03:37:43.661 INFO:teuthology.orchestra.run.smithi083.stdout: python3-toml noarch 0.10.2-6.el9 appstream 42 k 2024-09-14T03:37:43.661 INFO:teuthology.orchestra.run.smithi083.stdout: python3-urllib3 noarch 1.26.5-6.el9 baseos 215 k 2024-09-14T03:37:43.661 INFO:teuthology.orchestra.run.smithi083.stdout: python3-webob noarch 1.8.8-2.el9 epel 230 k 2024-09-14T03:37:43.661 INFO:teuthology.orchestra.run.smithi083.stdout: python3-werkzeug noarch 2.0.3-3.el9.1 epel 427 k 2024-09-14T03:37:43.662 INFO:teuthology.orchestra.run.smithi083.stdout: python3-zc-lockfile noarch 2.0-10.el9 epel 20 k 2024-09-14T03:37:43.662 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:37:43.662 INFO:teuthology.orchestra.run.smithi083.stdout:Transaction Summary 2024-09-14T03:37:43.662 INFO:teuthology.orchestra.run.smithi083.stdout:================================================================================ 2024-09-14T03:37:43.662 INFO:teuthology.orchestra.run.smithi083.stdout:Install 37 Packages 2024-09-14T03:37:43.662 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:37:43.662 INFO:teuthology.orchestra.run.smithi083.stdout:Total download size: 30 M 2024-09-14T03:37:43.663 INFO:teuthology.orchestra.run.smithi083.stdout:Installed size: 108 M 2024-09-14T03:37:43.663 INFO:teuthology.orchestra.run.smithi083.stdout:Downloading Packages: 2024-09-14T03:37:44.141 INFO:teuthology.orchestra.run.smithi083.stdout:(1/37): ceph-19.1.1-324.gecab7be5.el9.x86_64.rp 46 kB/s | 6.5 kB 00:00 2024-09-14T03:37:44.333 INFO:teuthology.orchestra.run.smithi083.stdout:(2/37): ceph-mgr-19.1.1-324.gecab7be5.el9.x86_6 4.6 MB/s | 1.5 MB 00:00 2024-09-14T03:37:44.375 INFO:teuthology.orchestra.run.smithi083.stdout:(3/37): ceph-mds-19.1.1-324.gecab7be5.el9.x86_6 6.4 MB/s | 2.4 MB 00:00 2024-09-14T03:37:44.442 INFO:teuthology.orchestra.run.smithi083.stdout:(4/37): ceph-mgr-modules-core-19.1.1-324.gecab7 3.6 MB/s | 249 kB 00:00 2024-09-14T03:37:44.567 INFO:teuthology.orchestra.run.smithi083.stdout:(5/37): python3-cffi-1.14.5-5.el9.x86_64.rpm 2.0 MB/s | 253 kB 00:00 2024-09-14T03:37:44.626 INFO:teuthology.orchestra.run.smithi083.stdout:(6/37): ceph-mon-19.1.1-324.gecab7be5.el9.x86_6 9.7 MB/s | 4.7 MB 00:00 2024-09-14T03:37:44.693 INFO:teuthology.orchestra.run.smithi083.stdout:(7/37): python3-cryptography-36.0.1-4.el9.x86_6 9.8 MB/s | 1.2 MB 00:00 2024-09-14T03:37:44.734 INFO:teuthology.orchestra.run.smithi083.stdout:(8/37): python3-pycparser-2.20-6.el9.noarch.rpm 3.2 MB/s | 135 kB 00:00 2024-09-14T03:37:44.760 INFO:teuthology.orchestra.run.smithi083.stdout:(9/37): python3-ply-3.11-14.el9.noarch.rpm 796 kB/s | 106 kB 00:00 2024-09-14T03:37:44.785 INFO:teuthology.orchestra.run.smithi083.stdout:(10/37): python3-requests-2.25.1-8.el9.noarch.r 2.4 MB/s | 125 kB 00:00 2024-09-14T03:37:44.819 INFO:teuthology.orchestra.run.smithi083.stdout:(11/37): python3-urllib3-1.26.5-6.el9.noarch.rp 3.6 MB/s | 215 kB 00:00 2024-09-14T03:37:45.027 INFO:teuthology.orchestra.run.smithi083.stdout:(12/37): ceph-osd-19.1.1-324.gecab7be5.el9.x86_ 24 MB/s | 17 MB 00:00 2024-09-14T03:37:45.603 INFO:teuthology.orchestra.run.smithi083.stdout:(13/37): python3-markupsafe-1.1.1-12.el9.x86_64 60 kB/s | 35 kB 00:00 2024-09-14T03:37:45.628 INFO:teuthology.orchestra.run.smithi083.stdout:(14/37): lua-5.4.4-4.el9.x86_64.rpm 224 kB/s | 188 kB 00:00 2024-09-14T03:37:45.670 INFO:teuthology.orchestra.run.smithi083.stdout:(15/37): python3-mako-1.1.4-6.el9.noarch.rpm 202 kB/s | 172 kB 00:00 2024-09-14T03:37:46.220 INFO:teuthology.orchestra.run.smithi083.stdout:(16/37): python3-pytz-2021.1-5.el9.noarch.rpm 86 kB/s | 51 kB 00:00 2024-09-14T03:37:46.246 INFO:teuthology.orchestra.run.smithi083.stdout:(17/37): python3-packaging-20.9-5.el9.noarch.rp 120 kB/s | 77 kB 00:00 2024-09-14T03:37:46.271 INFO:teuthology.orchestra.run.smithi083.stdout:(18/37): python3-toml-0.10.2-6.el9.noarch.rpm 69 kB/s | 42 kB 00:00 2024-09-14T03:37:46.308 INFO:teuthology.orchestra.run.smithi083.stdout:(19/37): lua-devel-5.4.4-4.el9.x86_64.rpm 262 kB/s | 22 kB 00:00 2024-09-14T03:37:46.366 INFO:teuthology.orchestra.run.smithi083.stdout:(20/37): luarocks-3.9.2-5.el9.noarch.rpm 1.6 MB/s | 151 kB 00:00 2024-09-14T03:37:46.392 INFO:teuthology.orchestra.run.smithi083.stdout:(21/37): python3-bcrypt-3.2.2-1.el9.x86_64.rpm 361 kB/s | 43 kB 00:00 2024-09-14T03:37:46.417 INFO:teuthology.orchestra.run.smithi083.stdout:(22/37): python3-jaraco-8.2.1-3.el9.noarch.rpm 431 kB/s | 11 kB 00:00 2024-09-14T03:37:46.442 INFO:teuthology.orchestra.run.smithi083.stdout:(23/37): python3-cheroot-8.6.0-4.el9.noarch.rpm 1.3 MB/s | 172 kB 00:00 2024-09-14T03:37:46.467 INFO:teuthology.orchestra.run.smithi083.stdout:(24/37): python3-jaraco-classes-3.2.1-5.el9.noa 352 kB/s | 18 kB 00:00 2024-09-14T03:37:46.493 INFO:teuthology.orchestra.run.smithi083.stdout:(25/37): python3-jaraco-collections-3.0.0-8.el9 461 kB/s | 23 kB 00:00 2024-09-14T03:37:46.518 INFO:teuthology.orchestra.run.smithi083.stdout:(26/37): python3-jaraco-functools-3.5.0-2.el9.n 387 kB/s | 19 kB 00:00 2024-09-14T03:37:46.543 INFO:teuthology.orchestra.run.smithi083.stdout:(27/37): python3-cherrypy-18.6.1-2.el9.noarch.r 2.0 MB/s | 358 kB 00:00 2024-09-14T03:37:46.560 INFO:teuthology.orchestra.run.smithi083.stdout:(28/37): python3-jaraco-text-3.2.0-6.el9.noarch 293 kB/s | 20 kB 00:00 2024-09-14T03:37:46.577 INFO:teuthology.orchestra.run.smithi083.stdout:(29/37): python3-logutils-0.3.5-21.el9.noarch.r 785 kB/s | 46 kB 00:00 2024-09-14T03:37:46.602 INFO:teuthology.orchestra.run.smithi083.stdout:(30/37): python3-more-itertools-8.12.0-2.el9.no 1.3 MB/s | 79 kB 00:00 2024-09-14T03:37:46.619 INFO:teuthology.orchestra.run.smithi083.stdout:(31/37): python3-portend-3.1.0-2.el9.noarch.rpm 393 kB/s | 16 kB 00:00 2024-09-14T03:37:46.653 INFO:teuthology.orchestra.run.smithi083.stdout:(32/37): python3-pyOpenSSL-21.0.0-1.el9.noarch. 1.7 MB/s | 90 kB 00:00 2024-09-14T03:37:46.678 INFO:teuthology.orchestra.run.smithi083.stdout:(33/37): python3-tempora-5.0.0-2.el9.noarch.rpm 610 kB/s | 36 kB 00:00 2024-09-14T03:37:46.712 INFO:teuthology.orchestra.run.smithi083.stdout:(34/37): python3-pecan-1.4.2-3.el9.noarch.rpm 1.8 MB/s | 272 kB 00:00 2024-09-14T03:37:46.737 INFO:teuthology.orchestra.run.smithi083.stdout:(35/37): python3-zc-lockfile-2.0-10.el9.noarch. 801 kB/s | 20 kB 00:00 2024-09-14T03:37:46.762 INFO:teuthology.orchestra.run.smithi083.stdout:(36/37): python3-webob-1.8.8-2.el9.noarch.rpm 2.7 MB/s | 230 kB 00:00 2024-09-14T03:37:46.788 INFO:teuthology.orchestra.run.smithi083.stdout:(37/37): python3-werkzeug-2.0.3-3.el9.1.noarch. 3.8 MB/s | 427 kB 00:00 2024-09-14T03:37:46.793 INFO:teuthology.orchestra.run.smithi083.stdout:-------------------------------------------------------------------------------- 2024-09-14T03:37:46.794 INFO:teuthology.orchestra.run.smithi083.stdout:Total 9.7 MB/s | 30 MB 00:03 2024-09-14T03:37:46.946 INFO:teuthology.orchestra.run.smithi083.stdout:Running transaction check 2024-09-14T03:37:46.987 INFO:teuthology.orchestra.run.smithi083.stdout:Transaction check succeeded. 2024-09-14T03:37:46.987 INFO:teuthology.orchestra.run.smithi083.stdout:Running transaction test 2024-09-14T03:37:47.481 INFO:teuthology.orchestra.run.smithi083.stdout:Transaction test succeeded. 2024-09-14T03:37:47.482 INFO:teuthology.orchestra.run.smithi083.stdout:Running transaction 2024-09-14T03:37:48.856 INFO:teuthology.orchestra.run.smithi083.stdout: Preparing : 1/1 2024-09-14T03:37:48.924 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : python3-more-itertools-8.12.0-2.el9.noarch 1/37 2024-09-14T03:37:48.976 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : python3-jaraco-8.2.1-3.el9.noarch 2/37 2024-09-14T03:37:49.010 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : lua-5.4.4-4.el9.x86_64 3/37 2024-09-14T03:37:49.083 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : lua-devel-5.4.4-4.el9.x86_64 4/37 2024-09-14T03:37:49.126 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : luarocks-3.9.2-5.el9.noarch 5/37 2024-09-14T03:37:49.158 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : python3-jaraco-classes-3.2.1-5.el9.noarch 6/37 2024-09-14T03:37:49.251 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : python3-zc-lockfile-2.0-10.el9.noarch 7/37 2024-09-14T03:37:49.329 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : python3-werkzeug-2.0.3-3.el9.1.noarch 8/37 2024-09-14T03:37:49.369 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : python3-webob-1.8.8-2.el9.noarch 9/37 2024-09-14T03:37:49.405 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : python3-logutils-0.3.5-21.el9.noarch 10/37 2024-09-14T03:37:49.435 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : python3-toml-0.10.2-6.el9.noarch 11/37 2024-09-14T03:37:49.469 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : python3-jaraco-functools-3.5.0-2.el9.noarch 12/37 2024-09-14T03:37:49.502 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : python3-jaraco-text-3.2.0-6.el9.noarch 13/37 2024-09-14T03:37:49.551 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : python3-jaraco-collections-3.0.0-8.el9.noarch 14/37 2024-09-14T03:37:49.595 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : python3-pytz-2021.1-5.el9.noarch 15/37 2024-09-14T03:37:49.629 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : python3-tempora-5.0.0-2.el9.noarch 16/37 2024-09-14T03:37:49.678 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : python3-portend-3.1.0-2.el9.noarch 17/37 2024-09-14T03:37:49.709 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : python3-packaging-20.9-5.el9.noarch 18/37 2024-09-14T03:37:49.783 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : python3-markupsafe-1.1.1-12.el9.x86_64 19/37 2024-09-14T03:37:49.931 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : python3-mako-1.1.4-6.el9.noarch 20/37 2024-09-14T03:37:50.012 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : python3-pecan-1.4.2-3.el9.noarch 21/37 2024-09-14T03:37:50.073 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : python3-urllib3-1.26.5-6.el9.noarch 22/37 2024-09-14T03:37:50.112 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : python3-requests-2.25.1-8.el9.noarch 23/37 2024-09-14T03:37:50.161 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : python3-ply-3.11-14.el9.noarch 24/37 2024-09-14T03:37:50.217 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : python3-pycparser-2.20-6.el9.noarch 25/37 2024-09-14T03:37:50.404 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : python3-cffi-1.14.5-5.el9.x86_64 26/37 2024-09-14T03:37:50.458 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : python3-cryptography-36.0.1-4.el9.x86_64 27/37 2024-09-14T03:37:50.536 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : python3-pyOpenSSL-21.0.0-1.el9.noarch 28/37 2024-09-14T03:37:50.625 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : python3-cheroot-8.6.0-4.el9.noarch 29/37 2024-09-14T03:37:50.665 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : python3-cherrypy-18.6.1-2.el9.noarch 30/37 2024-09-14T03:37:50.786 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : python3-bcrypt-3.2.2-1.el9.x86_64 31/37 2024-09-14T03:37:50.947 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : ceph-mgr-modules-core-2:19.1.1-324.gecab7be5.el9.n 32/37 2024-09-14T03:37:50.976 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : ceph-mgr-2:19.1.1-324.gecab7be5.el9.x86_64 33/37 2024-09-14T03:37:51.002 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: ceph-mgr-2:19.1.1-324.gecab7be5.el9.x86_64 33/37 2024-09-14T03:37:51.002 INFO:teuthology.orchestra.run.smithi083.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-14T03:37:51.002 INFO:teuthology.orchestra.run.smithi083.stdout:Invalid unit name "ceph-mgr@*.service" escaped as "ceph-mgr@\x2a.service". 2024-09-14T03:37:51.002 INFO:teuthology.orchestra.run.smithi083.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-09-14T03:37:51.002 INFO:teuthology.orchestra.run.smithi083.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-09-14T03:37:51.002 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:37:52.435 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : ceph-osd-2:19.1.1-324.gecab7be5.el9.x86_64 34/37 2024-09-14T03:37:52.462 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: ceph-osd-2:19.1.1-324.gecab7be5.el9.x86_64 34/37 2024-09-14T03:37:52.462 INFO:teuthology.orchestra.run.smithi083.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-14T03:37:52.462 INFO:teuthology.orchestra.run.smithi083.stdout:Invalid unit name "ceph-osd@*.service" escaped as "ceph-osd@\x2a.service". 2024-09-14T03:37:52.462 INFO:teuthology.orchestra.run.smithi083.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-09-14T03:37:52.462 INFO:teuthology.orchestra.run.smithi083.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-09-14T03:37:52.462 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:37:52.902 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : ceph-mon-2:19.1.1-324.gecab7be5.el9.x86_64 35/37 2024-09-14T03:37:52.926 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: ceph-mon-2:19.1.1-324.gecab7be5.el9.x86_64 35/37 2024-09-14T03:37:52.926 INFO:teuthology.orchestra.run.smithi083.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-14T03:37:52.926 INFO:teuthology.orchestra.run.smithi083.stdout:Invalid unit name "ceph-mon@*.service" escaped as "ceph-mon@\x2a.service". 2024-09-14T03:37:52.926 INFO:teuthology.orchestra.run.smithi083.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-09-14T03:37:52.927 INFO:teuthology.orchestra.run.smithi083.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-09-14T03:37:52.927 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:37:53.153 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : ceph-mds-2:19.1.1-324.gecab7be5.el9.x86_64 36/37 2024-09-14T03:37:53.176 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: ceph-mds-2:19.1.1-324.gecab7be5.el9.x86_64 36/37 2024-09-14T03:37:53.176 INFO:teuthology.orchestra.run.smithi083.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-14T03:37:53.176 INFO:teuthology.orchestra.run.smithi083.stdout:Invalid unit name "ceph-mds@*.service" escaped as "ceph-mds@\x2a.service". 2024-09-14T03:37:53.176 INFO:teuthology.orchestra.run.smithi083.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-09-14T03:37:53.177 INFO:teuthology.orchestra.run.smithi083.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-09-14T03:37:53.177 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:37:53.252 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : ceph-2:19.1.1-324.gecab7be5.el9.x86_64 37/37 2024-09-14T03:37:55.021 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: ceph-2:19.1.1-324.gecab7be5.el9.x86_64 37/37 2024-09-14T03:37:55.021 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : ceph-2:19.1.1-324.gecab7be5.el9.x86_64 1/37 2024-09-14T03:37:55.021 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : ceph-mds-2:19.1.1-324.gecab7be5.el9.x86_64 2/37 2024-09-14T03:37:55.021 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : ceph-mgr-2:19.1.1-324.gecab7be5.el9.x86_64 3/37 2024-09-14T03:37:55.021 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : ceph-mon-2:19.1.1-324.gecab7be5.el9.x86_64 4/37 2024-09-14T03:37:55.022 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : ceph-osd-2:19.1.1-324.gecab7be5.el9.x86_64 5/37 2024-09-14T03:37:55.022 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : ceph-mgr-modules-core-2:19.1.1-324.gecab7be5.el9.n 6/37 2024-09-14T03:37:55.022 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : python3-cffi-1.14.5-5.el9.x86_64 7/37 2024-09-14T03:37:55.022 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : python3-cryptography-36.0.1-4.el9.x86_64 8/37 2024-09-14T03:37:55.022 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : python3-ply-3.11-14.el9.noarch 9/37 2024-09-14T03:37:55.022 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : python3-pycparser-2.20-6.el9.noarch 10/37 2024-09-14T03:37:55.022 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : python3-requests-2.25.1-8.el9.noarch 11/37 2024-09-14T03:37:55.022 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : python3-urllib3-1.26.5-6.el9.noarch 12/37 2024-09-14T03:37:55.022 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : lua-5.4.4-4.el9.x86_64 13/37 2024-09-14T03:37:55.022 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : python3-mako-1.1.4-6.el9.noarch 14/37 2024-09-14T03:37:55.022 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : python3-markupsafe-1.1.1-12.el9.x86_64 15/37 2024-09-14T03:37:55.022 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : python3-packaging-20.9-5.el9.noarch 16/37 2024-09-14T03:37:55.022 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : python3-pytz-2021.1-5.el9.noarch 17/37 2024-09-14T03:37:55.023 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : python3-toml-0.10.2-6.el9.noarch 18/37 2024-09-14T03:37:55.023 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : lua-devel-5.4.4-4.el9.x86_64 19/37 2024-09-14T03:37:55.023 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : luarocks-3.9.2-5.el9.noarch 20/37 2024-09-14T03:37:55.023 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : python3-bcrypt-3.2.2-1.el9.x86_64 21/37 2024-09-14T03:37:55.023 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : python3-cheroot-8.6.0-4.el9.noarch 22/37 2024-09-14T03:37:55.023 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : python3-cherrypy-18.6.1-2.el9.noarch 23/37 2024-09-14T03:37:55.023 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : python3-jaraco-8.2.1-3.el9.noarch 24/37 2024-09-14T03:37:55.023 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : python3-jaraco-classes-3.2.1-5.el9.noarch 25/37 2024-09-14T03:37:55.023 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : python3-jaraco-collections-3.0.0-8.el9.noarch 26/37 2024-09-14T03:37:55.023 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : python3-jaraco-functools-3.5.0-2.el9.noarch 27/37 2024-09-14T03:37:55.023 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : python3-jaraco-text-3.2.0-6.el9.noarch 28/37 2024-09-14T03:37:55.024 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : python3-logutils-0.3.5-21.el9.noarch 29/37 2024-09-14T03:37:55.024 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : python3-more-itertools-8.12.0-2.el9.noarch 30/37 2024-09-14T03:37:55.024 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : python3-pecan-1.4.2-3.el9.noarch 31/37 2024-09-14T03:37:55.025 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : python3-portend-3.1.0-2.el9.noarch 32/37 2024-09-14T03:37:55.025 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : python3-pyOpenSSL-21.0.0-1.el9.noarch 33/37 2024-09-14T03:37:55.025 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : python3-tempora-5.0.0-2.el9.noarch 34/37 2024-09-14T03:37:55.025 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : python3-webob-1.8.8-2.el9.noarch 35/37 2024-09-14T03:37:55.025 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : python3-werkzeug-2.0.3-3.el9.1.noarch 36/37 2024-09-14T03:37:56.140 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : python3-zc-lockfile-2.0-10.el9.noarch 37/37 2024-09-14T03:37:56.141 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:37:56.141 INFO:teuthology.orchestra.run.smithi083.stdout:Installed: 2024-09-14T03:37:56.141 INFO:teuthology.orchestra.run.smithi083.stdout: ceph-2:19.1.1-324.gecab7be5.el9.x86_64 2024-09-14T03:37:56.141 INFO:teuthology.orchestra.run.smithi083.stdout: ceph-mds-2:19.1.1-324.gecab7be5.el9.x86_64 2024-09-14T03:37:56.141 INFO:teuthology.orchestra.run.smithi083.stdout: ceph-mgr-2:19.1.1-324.gecab7be5.el9.x86_64 2024-09-14T03:37:56.141 INFO:teuthology.orchestra.run.smithi083.stdout: ceph-mgr-modules-core-2:19.1.1-324.gecab7be5.el9.noarch 2024-09-14T03:37:56.141 INFO:teuthology.orchestra.run.smithi083.stdout: ceph-mon-2:19.1.1-324.gecab7be5.el9.x86_64 2024-09-14T03:37:56.141 INFO:teuthology.orchestra.run.smithi083.stdout: ceph-osd-2:19.1.1-324.gecab7be5.el9.x86_64 2024-09-14T03:37:56.141 INFO:teuthology.orchestra.run.smithi083.stdout: lua-5.4.4-4.el9.x86_64 2024-09-14T03:37:56.141 INFO:teuthology.orchestra.run.smithi083.stdout: lua-devel-5.4.4-4.el9.x86_64 2024-09-14T03:37:56.141 INFO:teuthology.orchestra.run.smithi083.stdout: luarocks-3.9.2-5.el9.noarch 2024-09-14T03:37:56.141 INFO:teuthology.orchestra.run.smithi083.stdout: python3-bcrypt-3.2.2-1.el9.x86_64 2024-09-14T03:37:56.142 INFO:teuthology.orchestra.run.smithi083.stdout: python3-cffi-1.14.5-5.el9.x86_64 2024-09-14T03:37:56.142 INFO:teuthology.orchestra.run.smithi083.stdout: python3-cheroot-8.6.0-4.el9.noarch 2024-09-14T03:37:56.142 INFO:teuthology.orchestra.run.smithi083.stdout: python3-cherrypy-18.6.1-2.el9.noarch 2024-09-14T03:37:56.142 INFO:teuthology.orchestra.run.smithi083.stdout: python3-cryptography-36.0.1-4.el9.x86_64 2024-09-14T03:37:56.142 INFO:teuthology.orchestra.run.smithi083.stdout: python3-jaraco-8.2.1-3.el9.noarch 2024-09-14T03:37:56.142 INFO:teuthology.orchestra.run.smithi083.stdout: python3-jaraco-classes-3.2.1-5.el9.noarch 2024-09-14T03:37:56.142 INFO:teuthology.orchestra.run.smithi083.stdout: python3-jaraco-collections-3.0.0-8.el9.noarch 2024-09-14T03:37:56.142 INFO:teuthology.orchestra.run.smithi083.stdout: python3-jaraco-functools-3.5.0-2.el9.noarch 2024-09-14T03:37:56.142 INFO:teuthology.orchestra.run.smithi083.stdout: python3-jaraco-text-3.2.0-6.el9.noarch 2024-09-14T03:37:56.142 INFO:teuthology.orchestra.run.smithi083.stdout: python3-logutils-0.3.5-21.el9.noarch 2024-09-14T03:37:56.142 INFO:teuthology.orchestra.run.smithi083.stdout: python3-mako-1.1.4-6.el9.noarch 2024-09-14T03:37:56.142 INFO:teuthology.orchestra.run.smithi083.stdout: python3-markupsafe-1.1.1-12.el9.x86_64 2024-09-14T03:37:56.142 INFO:teuthology.orchestra.run.smithi083.stdout: python3-more-itertools-8.12.0-2.el9.noarch 2024-09-14T03:37:56.142 INFO:teuthology.orchestra.run.smithi083.stdout: python3-packaging-20.9-5.el9.noarch 2024-09-14T03:37:56.142 INFO:teuthology.orchestra.run.smithi083.stdout: python3-pecan-1.4.2-3.el9.noarch 2024-09-14T03:37:56.143 INFO:teuthology.orchestra.run.smithi083.stdout: python3-ply-3.11-14.el9.noarch 2024-09-14T03:37:56.143 INFO:teuthology.orchestra.run.smithi083.stdout: python3-portend-3.1.0-2.el9.noarch 2024-09-14T03:37:56.143 INFO:teuthology.orchestra.run.smithi083.stdout: python3-pyOpenSSL-21.0.0-1.el9.noarch 2024-09-14T03:37:56.143 INFO:teuthology.orchestra.run.smithi083.stdout: python3-pycparser-2.20-6.el9.noarch 2024-09-14T03:37:56.143 INFO:teuthology.orchestra.run.smithi083.stdout: python3-pytz-2021.1-5.el9.noarch 2024-09-14T03:37:56.145 INFO:teuthology.orchestra.run.smithi083.stdout: python3-requests-2.25.1-8.el9.noarch 2024-09-14T03:37:56.145 INFO:teuthology.orchestra.run.smithi083.stdout: python3-tempora-5.0.0-2.el9.noarch 2024-09-14T03:37:56.145 INFO:teuthology.orchestra.run.smithi083.stdout: python3-toml-0.10.2-6.el9.noarch 2024-09-14T03:37:56.145 INFO:teuthology.orchestra.run.smithi083.stdout: python3-urllib3-1.26.5-6.el9.noarch 2024-09-14T03:37:56.146 INFO:teuthology.orchestra.run.smithi083.stdout: python3-webob-1.8.8-2.el9.noarch 2024-09-14T03:37:56.146 INFO:teuthology.orchestra.run.smithi083.stdout: python3-werkzeug-2.0.3-3.el9.1.noarch 2024-09-14T03:37:56.146 INFO:teuthology.orchestra.run.smithi083.stdout: python3-zc-lockfile-2.0-10.el9.noarch 2024-09-14T03:37:56.146 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:37:56.146 INFO:teuthology.orchestra.run.smithi083.stdout:Complete! 2024-09-14T03:37:56.509 DEBUG:teuthology.orchestra.run.smithi083:> sudo yum -y install ceph-base 2024-09-14T03:37:57.078 INFO:teuthology.orchestra.run.smithi083.stdout:Last metadata expiration check: 0:00:53 ago on Sat 14 Sep 2024 03:37:04 AM UTC. 2024-09-14T03:37:57.168 INFO:teuthology.orchestra.run.smithi083.stdout:Package ceph-base-2:19.1.1-324.gecab7be5.el9.x86_64 is already installed. 2024-09-14T03:37:57.209 INFO:teuthology.orchestra.run.smithi083.stdout:Dependencies resolved. 2024-09-14T03:37:57.210 INFO:teuthology.orchestra.run.smithi083.stdout:Nothing to do. 2024-09-14T03:37:57.210 INFO:teuthology.orchestra.run.smithi083.stdout:Complete! 2024-09-14T03:37:57.272 DEBUG:teuthology.orchestra.run.smithi083:> sudo yum -y install ceph-immutable-object-cache 2024-09-14T03:37:57.829 INFO:teuthology.orchestra.run.smithi083.stdout:Last metadata expiration check: 0:00:53 ago on Sat 14 Sep 2024 03:37:04 AM UTC. 2024-09-14T03:37:57.960 INFO:teuthology.orchestra.run.smithi083.stdout:Dependencies resolved. 2024-09-14T03:37:57.961 INFO:teuthology.orchestra.run.smithi083.stdout:================================================================================ 2024-09-14T03:37:57.961 INFO:teuthology.orchestra.run.smithi083.stdout: Package Arch Version Repo Size 2024-09-14T03:37:57.961 INFO:teuthology.orchestra.run.smithi083.stdout:================================================================================ 2024-09-14T03:37:57.961 INFO:teuthology.orchestra.run.smithi083.stdout:Installing: 2024-09-14T03:37:57.961 INFO:teuthology.orchestra.run.smithi083.stdout: ceph-immutable-object-cache x86_64 2:19.1.1-324.gecab7be5.el9 ceph 145 k 2024-09-14T03:37:57.961 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:37:57.961 INFO:teuthology.orchestra.run.smithi083.stdout:Transaction Summary 2024-09-14T03:37:57.961 INFO:teuthology.orchestra.run.smithi083.stdout:================================================================================ 2024-09-14T03:37:57.962 INFO:teuthology.orchestra.run.smithi083.stdout:Install 1 Package 2024-09-14T03:37:57.962 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:37:57.962 INFO:teuthology.orchestra.run.smithi083.stdout:Total download size: 145 k 2024-09-14T03:37:57.962 INFO:teuthology.orchestra.run.smithi083.stdout:Installed size: 435 k 2024-09-14T03:37:57.962 INFO:teuthology.orchestra.run.smithi083.stdout:Downloading Packages: 2024-09-14T03:37:58.179 INFO:teuthology.orchestra.run.smithi083.stdout:ceph-immutable-object-cache-19.1.1-324.gecab7be 669 kB/s | 145 kB 00:00 2024-09-14T03:37:58.179 INFO:teuthology.orchestra.run.smithi083.stdout:-------------------------------------------------------------------------------- 2024-09-14T03:37:58.180 INFO:teuthology.orchestra.run.smithi083.stdout:Total 664 kB/s | 145 kB 00:00 2024-09-14T03:37:58.180 INFO:teuthology.orchestra.run.smithi083.stdout:Running transaction check 2024-09-14T03:37:58.187 INFO:teuthology.orchestra.run.smithi083.stdout:Transaction check succeeded. 2024-09-14T03:37:58.187 INFO:teuthology.orchestra.run.smithi083.stdout:Running transaction test 2024-09-14T03:37:58.244 INFO:teuthology.orchestra.run.smithi083.stdout:Transaction test succeeded. 2024-09-14T03:37:58.244 INFO:teuthology.orchestra.run.smithi083.stdout:Running transaction 2024-09-14T03:37:58.387 INFO:teuthology.orchestra.run.smithi083.stdout: Preparing : 1/1 2024-09-14T03:37:58.463 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : ceph-immutable-object-cache-2:19.1.1-324.gecab7be5.e 1/1 2024-09-14T03:37:58.487 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: ceph-immutable-object-cache-2:19.1.1-324.gecab7be5.e 1/1 2024-09-14T03:37:58.488 INFO:teuthology.orchestra.run.smithi083.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-14T03:37:58.488 INFO:teuthology.orchestra.run.smithi083.stdout:Invalid unit name "ceph-immutable-object-cache@*.service" escaped as "ceph-immutable-object-cache@\x2a.service". 2024-09-14T03:37:58.488 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:37:59.506 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : ceph-immutable-object-cache-2:19.1.1-324.gecab7be5.e 1/1 2024-09-14T03:37:59.506 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:37:59.506 INFO:teuthology.orchestra.run.smithi083.stdout:Installed: 2024-09-14T03:37:59.506 INFO:teuthology.orchestra.run.smithi083.stdout: ceph-immutable-object-cache-2:19.1.1-324.gecab7be5.el9.x86_64 2024-09-14T03:37:59.506 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:37:59.506 INFO:teuthology.orchestra.run.smithi083.stdout:Complete! 2024-09-14T03:37:59.723 DEBUG:teuthology.orchestra.run.smithi083:> sudo yum -y install ceph-mgr 2024-09-14T03:38:00.304 INFO:teuthology.orchestra.run.smithi083.stdout:Last metadata expiration check: 0:00:56 ago on Sat 14 Sep 2024 03:37:04 AM UTC. 2024-09-14T03:38:00.395 INFO:teuthology.orchestra.run.smithi083.stdout:Package ceph-mgr-2:19.1.1-324.gecab7be5.el9.x86_64 is already installed. 2024-09-14T03:38:00.437 INFO:teuthology.orchestra.run.smithi083.stdout:Dependencies resolved. 2024-09-14T03:38:00.438 INFO:teuthology.orchestra.run.smithi083.stdout:Nothing to do. 2024-09-14T03:38:00.438 INFO:teuthology.orchestra.run.smithi083.stdout:Complete! 2024-09-14T03:38:00.504 DEBUG:teuthology.orchestra.run.smithi083:> sudo yum -y install ceph-fuse 2024-09-14T03:38:01.063 INFO:teuthology.orchestra.run.smithi083.stdout:Last metadata expiration check: 0:00:57 ago on Sat 14 Sep 2024 03:37:04 AM UTC. 2024-09-14T03:38:01.200 INFO:teuthology.orchestra.run.smithi083.stdout:Dependencies resolved. 2024-09-14T03:38:01.200 INFO:teuthology.orchestra.run.smithi083.stdout:================================================================================ 2024-09-14T03:38:01.200 INFO:teuthology.orchestra.run.smithi083.stdout: Package Arch Version Repository Size 2024-09-14T03:38:01.200 INFO:teuthology.orchestra.run.smithi083.stdout:================================================================================ 2024-09-14T03:38:01.200 INFO:teuthology.orchestra.run.smithi083.stdout:Installing: 2024-09-14T03:38:01.201 INFO:teuthology.orchestra.run.smithi083.stdout: ceph-fuse x86_64 2:19.1.1-324.gecab7be5.el9 ceph 896 k 2024-09-14T03:38:01.201 INFO:teuthology.orchestra.run.smithi083.stdout:Installing dependencies: 2024-09-14T03:38:01.201 INFO:teuthology.orchestra.run.smithi083.stdout: fuse x86_64 2.9.9-16.el9 baseos 80 k 2024-09-14T03:38:01.201 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:38:01.201 INFO:teuthology.orchestra.run.smithi083.stdout:Transaction Summary 2024-09-14T03:38:01.201 INFO:teuthology.orchestra.run.smithi083.stdout:================================================================================ 2024-09-14T03:38:01.201 INFO:teuthology.orchestra.run.smithi083.stdout:Install 2 Packages 2024-09-14T03:38:01.201 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:38:01.202 INFO:teuthology.orchestra.run.smithi083.stdout:Total download size: 976 k 2024-09-14T03:38:01.202 INFO:teuthology.orchestra.run.smithi083.stdout:Installed size: 2.9 M 2024-09-14T03:38:01.202 INFO:teuthology.orchestra.run.smithi083.stdout:Downloading Packages: 2024-09-14T03:38:01.583 INFO:teuthology.orchestra.run.smithi083.stdout:(1/2): ceph-fuse-19.1.1-324.gecab7be5.el9.x86_6 3.4 MB/s | 896 kB 00:00 2024-09-14T03:38:01.783 INFO:teuthology.orchestra.run.smithi083.stdout:(2/2): fuse-2.9.9-16.el9.x86_64.rpm 176 kB/s | 80 kB 00:00 2024-09-14T03:38:01.783 INFO:teuthology.orchestra.run.smithi083.stdout:-------------------------------------------------------------------------------- 2024-09-14T03:38:01.784 INFO:teuthology.orchestra.run.smithi083.stdout:Total 1.6 MB/s | 976 kB 00:00 2024-09-14T03:38:01.795 INFO:teuthology.orchestra.run.smithi083.stdout:Running transaction check 2024-09-14T03:38:01.817 INFO:teuthology.orchestra.run.smithi083.stdout:Transaction check succeeded. 2024-09-14T03:38:01.817 INFO:teuthology.orchestra.run.smithi083.stdout:Running transaction test 2024-09-14T03:38:01.890 INFO:teuthology.orchestra.run.smithi083.stdout:Transaction test succeeded. 2024-09-14T03:38:01.890 INFO:teuthology.orchestra.run.smithi083.stdout:Running transaction 2024-09-14T03:38:02.079 INFO:teuthology.orchestra.run.smithi083.stdout: Preparing : 1/1 2024-09-14T03:38:02.210 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : fuse-2.9.9-16.el9.x86_64 1/2 2024-09-14T03:38:02.272 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : ceph-fuse-2:19.1.1-324.gecab7be5.el9.x86_64 2/2 2024-09-14T03:38:03.030 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: ceph-fuse-2:19.1.1-324.gecab7be5.el9.x86_64 2/2 2024-09-14T03:38:03.030 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : ceph-fuse-2:19.1.1-324.gecab7be5.el9.x86_64 1/2 2024-09-14T03:38:03.312 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : fuse-2.9.9-16.el9.x86_64 2/2 2024-09-14T03:38:03.312 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:38:03.312 INFO:teuthology.orchestra.run.smithi083.stdout:Installed: 2024-09-14T03:38:03.312 INFO:teuthology.orchestra.run.smithi083.stdout: ceph-fuse-2:19.1.1-324.gecab7be5.el9.x86_64 fuse-2.9.9-16.el9.x86_64 2024-09-14T03:38:03.313 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:38:03.313 INFO:teuthology.orchestra.run.smithi083.stdout:Complete! 2024-09-14T03:38:03.538 DEBUG:teuthology.orchestra.run.smithi083:> sudo yum -y install librados-devel 2024-09-14T03:38:04.111 INFO:teuthology.orchestra.run.smithi083.stdout:Last metadata expiration check: 0:01:00 ago on Sat 14 Sep 2024 03:37:04 AM UTC. 2024-09-14T03:38:04.243 INFO:teuthology.orchestra.run.smithi083.stdout:Dependencies resolved. 2024-09-14T03:38:04.244 INFO:teuthology.orchestra.run.smithi083.stdout:================================================================================ 2024-09-14T03:38:04.244 INFO:teuthology.orchestra.run.smithi083.stdout: Package Arch Version Repo Size 2024-09-14T03:38:04.244 INFO:teuthology.orchestra.run.smithi083.stdout:================================================================================ 2024-09-14T03:38:04.244 INFO:teuthology.orchestra.run.smithi083.stdout:Installing: 2024-09-14T03:38:04.244 INFO:teuthology.orchestra.run.smithi083.stdout: librados-devel x86_64 2:19.1.1-324.gecab7be5.el9 ceph 127 k 2024-09-14T03:38:04.244 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:38:04.245 INFO:teuthology.orchestra.run.smithi083.stdout:Transaction Summary 2024-09-14T03:38:04.245 INFO:teuthology.orchestra.run.smithi083.stdout:================================================================================ 2024-09-14T03:38:04.245 INFO:teuthology.orchestra.run.smithi083.stdout:Install 1 Package 2024-09-14T03:38:04.245 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:38:04.245 INFO:teuthology.orchestra.run.smithi083.stdout:Total download size: 127 k 2024-09-14T03:38:04.245 INFO:teuthology.orchestra.run.smithi083.stdout:Installed size: 456 k 2024-09-14T03:38:04.245 INFO:teuthology.orchestra.run.smithi083.stdout:Downloading Packages: 2024-09-14T03:38:04.454 INFO:teuthology.orchestra.run.smithi083.stdout:librados-devel-19.1.1-324.gecab7be5.el9.x86_64. 606 kB/s | 127 kB 00:00 2024-09-14T03:38:04.454 INFO:teuthology.orchestra.run.smithi083.stdout:-------------------------------------------------------------------------------- 2024-09-14T03:38:04.455 INFO:teuthology.orchestra.run.smithi083.stdout:Total 603 kB/s | 127 kB 00:00 2024-09-14T03:38:04.455 INFO:teuthology.orchestra.run.smithi083.stdout:Running transaction check 2024-09-14T03:38:04.460 INFO:teuthology.orchestra.run.smithi083.stdout:Transaction check succeeded. 2024-09-14T03:38:04.460 INFO:teuthology.orchestra.run.smithi083.stdout:Running transaction test 2024-09-14T03:38:04.518 INFO:teuthology.orchestra.run.smithi083.stdout:Transaction test succeeded. 2024-09-14T03:38:04.518 INFO:teuthology.orchestra.run.smithi083.stdout:Running transaction 2024-09-14T03:38:04.671 INFO:teuthology.orchestra.run.smithi083.stdout: Preparing : 1/1 2024-09-14T03:38:04.767 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : librados-devel-2:19.1.1-324.gecab7be5.el9.x86_64 1/1 2024-09-14T03:38:05.171 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: librados-devel-2:19.1.1-324.gecab7be5.el9.x86_64 1/1 2024-09-14T03:38:05.447 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : librados-devel-2:19.1.1-324.gecab7be5.el9.x86_64 1/1 2024-09-14T03:38:05.448 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:38:05.448 INFO:teuthology.orchestra.run.smithi083.stdout:Installed: 2024-09-14T03:38:05.448 INFO:teuthology.orchestra.run.smithi083.stdout: librados-devel-2:19.1.1-324.gecab7be5.el9.x86_64 2024-09-14T03:38:05.448 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:38:05.448 INFO:teuthology.orchestra.run.smithi083.stdout:Complete! 2024-09-14T03:38:05.698 DEBUG:teuthology.orchestra.run.smithi083:> sudo yum -y install libcephfs2 2024-09-14T03:38:06.261 INFO:teuthology.orchestra.run.smithi083.stdout:Last metadata expiration check: 0:01:02 ago on Sat 14 Sep 2024 03:37:04 AM UTC. 2024-09-14T03:38:06.353 INFO:teuthology.orchestra.run.smithi083.stdout:Package libcephfs2-2:19.1.1-324.gecab7be5.el9.x86_64 is already installed. 2024-09-14T03:38:06.396 INFO:teuthology.orchestra.run.smithi083.stdout:Dependencies resolved. 2024-09-14T03:38:06.397 INFO:teuthology.orchestra.run.smithi083.stdout:Nothing to do. 2024-09-14T03:38:06.397 INFO:teuthology.orchestra.run.smithi083.stdout:Complete! 2024-09-14T03:38:06.462 DEBUG:teuthology.orchestra.run.smithi083:> sudo yum -y install libcephfs-devel 2024-09-14T03:38:07.023 INFO:teuthology.orchestra.run.smithi083.stdout:Last metadata expiration check: 0:01:03 ago on Sat 14 Sep 2024 03:37:04 AM UTC. 2024-09-14T03:38:07.155 INFO:teuthology.orchestra.run.smithi083.stdout:Dependencies resolved. 2024-09-14T03:38:07.155 INFO:teuthology.orchestra.run.smithi083.stdout:================================================================================ 2024-09-14T03:38:07.156 INFO:teuthology.orchestra.run.smithi083.stdout: Package Arch Version Repo Size 2024-09-14T03:38:07.156 INFO:teuthology.orchestra.run.smithi083.stdout:================================================================================ 2024-09-14T03:38:07.156 INFO:teuthology.orchestra.run.smithi083.stdout:Installing: 2024-09-14T03:38:07.156 INFO:teuthology.orchestra.run.smithi083.stdout: libcephfs-devel x86_64 2:19.1.1-324.gecab7be5.el9 ceph 32 k 2024-09-14T03:38:07.156 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:38:07.156 INFO:teuthology.orchestra.run.smithi083.stdout:Transaction Summary 2024-09-14T03:38:07.156 INFO:teuthology.orchestra.run.smithi083.stdout:================================================================================ 2024-09-14T03:38:07.156 INFO:teuthology.orchestra.run.smithi083.stdout:Install 1 Package 2024-09-14T03:38:07.156 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:38:07.156 INFO:teuthology.orchestra.run.smithi083.stdout:Total download size: 32 k 2024-09-14T03:38:07.156 INFO:teuthology.orchestra.run.smithi083.stdout:Installed size: 142 k 2024-09-14T03:38:07.156 INFO:teuthology.orchestra.run.smithi083.stdout:Downloading Packages: 2024-09-14T03:38:07.316 INFO:teuthology.orchestra.run.smithi083.stdout:libcephfs-devel-19.1.1-324.gecab7be5.el9.x86_64 198 kB/s | 32 kB 00:00 2024-09-14T03:38:07.317 INFO:teuthology.orchestra.run.smithi083.stdout:-------------------------------------------------------------------------------- 2024-09-14T03:38:07.317 INFO:teuthology.orchestra.run.smithi083.stdout:Total 196 kB/s | 32 kB 00:00 2024-09-14T03:38:07.317 INFO:teuthology.orchestra.run.smithi083.stdout:Running transaction check 2024-09-14T03:38:07.321 INFO:teuthology.orchestra.run.smithi083.stdout:Transaction check succeeded. 2024-09-14T03:38:07.321 INFO:teuthology.orchestra.run.smithi083.stdout:Running transaction test 2024-09-14T03:38:07.340 INFO:teuthology.orchestra.run.smithi083.stdout:Transaction test succeeded. 2024-09-14T03:38:07.340 INFO:teuthology.orchestra.run.smithi083.stdout:Running transaction 2024-09-14T03:38:07.426 INFO:teuthology.orchestra.run.smithi083.stdout: Preparing : 1/1 2024-09-14T03:38:07.521 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : libcephfs-devel-2:19.1.1-324.gecab7be5.el9.x86_64 1/1 2024-09-14T03:38:07.873 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: libcephfs-devel-2:19.1.1-324.gecab7be5.el9.x86_64 1/1 2024-09-14T03:38:08.096 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : libcephfs-devel-2:19.1.1-324.gecab7be5.el9.x86_64 1/1 2024-09-14T03:38:08.096 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:38:08.096 INFO:teuthology.orchestra.run.smithi083.stdout:Installed: 2024-09-14T03:38:08.096 INFO:teuthology.orchestra.run.smithi083.stdout: libcephfs-devel-2:19.1.1-324.gecab7be5.el9.x86_64 2024-09-14T03:38:08.096 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:38:08.097 INFO:teuthology.orchestra.run.smithi083.stdout:Complete! 2024-09-14T03:38:08.263 DEBUG:teuthology.orchestra.run.smithi083:> sudo yum -y install librados2 2024-09-14T03:38:08.823 INFO:teuthology.orchestra.run.smithi083.stdout:Last metadata expiration check: 0:01:04 ago on Sat 14 Sep 2024 03:37:04 AM UTC. 2024-09-14T03:38:08.913 INFO:teuthology.orchestra.run.smithi083.stdout:Package librados2-2:19.1.1-324.gecab7be5.el9.x86_64 is already installed. 2024-09-14T03:38:08.956 INFO:teuthology.orchestra.run.smithi083.stdout:Dependencies resolved. 2024-09-14T03:38:08.957 INFO:teuthology.orchestra.run.smithi083.stdout:Nothing to do. 2024-09-14T03:38:08.957 INFO:teuthology.orchestra.run.smithi083.stdout:Complete! 2024-09-14T03:38:09.030 DEBUG:teuthology.orchestra.run.smithi083:> sudo yum -y install librbd1 2024-09-14T03:38:09.586 INFO:teuthology.orchestra.run.smithi083.stdout:Last metadata expiration check: 0:01:05 ago on Sat 14 Sep 2024 03:37:04 AM UTC. 2024-09-14T03:38:09.674 INFO:teuthology.orchestra.run.smithi083.stdout:Package librbd1-2:19.1.1-324.gecab7be5.el9.x86_64 is already installed. 2024-09-14T03:38:09.716 INFO:teuthology.orchestra.run.smithi083.stdout:Dependencies resolved. 2024-09-14T03:38:09.717 INFO:teuthology.orchestra.run.smithi083.stdout:Nothing to do. 2024-09-14T03:38:09.717 INFO:teuthology.orchestra.run.smithi083.stdout:Complete! 2024-09-14T03:38:09.775 DEBUG:teuthology.orchestra.run.smithi083:> sudo yum -y install python3-rados 2024-09-14T03:38:10.328 INFO:teuthology.orchestra.run.smithi083.stdout:Last metadata expiration check: 0:01:06 ago on Sat 14 Sep 2024 03:37:04 AM UTC. 2024-09-14T03:38:10.417 INFO:teuthology.orchestra.run.smithi083.stdout:Package python3-rados-2:19.1.1-324.gecab7be5.el9.x86_64 is already installed. 2024-09-14T03:38:10.459 INFO:teuthology.orchestra.run.smithi083.stdout:Dependencies resolved. 2024-09-14T03:38:10.460 INFO:teuthology.orchestra.run.smithi083.stdout:Nothing to do. 2024-09-14T03:38:10.460 INFO:teuthology.orchestra.run.smithi083.stdout:Complete! 2024-09-14T03:38:10.517 DEBUG:teuthology.orchestra.run.smithi083:> sudo yum -y install python3-rgw 2024-09-14T03:38:11.075 INFO:teuthology.orchestra.run.smithi083.stdout:Last metadata expiration check: 0:01:07 ago on Sat 14 Sep 2024 03:37:04 AM UTC. 2024-09-14T03:38:11.166 INFO:teuthology.orchestra.run.smithi083.stdout:Package python3-rgw-2:19.1.1-324.gecab7be5.el9.x86_64 is already installed. 2024-09-14T03:38:11.209 INFO:teuthology.orchestra.run.smithi083.stdout:Dependencies resolved. 2024-09-14T03:38:11.210 INFO:teuthology.orchestra.run.smithi083.stdout:Nothing to do. 2024-09-14T03:38:11.210 INFO:teuthology.orchestra.run.smithi083.stdout:Complete! 2024-09-14T03:38:11.270 DEBUG:teuthology.orchestra.run.smithi083:> sudo yum -y install python3-cephfs 2024-09-14T03:38:11.835 INFO:teuthology.orchestra.run.smithi083.stdout:Last metadata expiration check: 0:01:07 ago on Sat 14 Sep 2024 03:37:04 AM UTC. 2024-09-14T03:38:11.926 INFO:teuthology.orchestra.run.smithi083.stdout:Package python3-cephfs-2:19.1.1-324.gecab7be5.el9.x86_64 is already installed. 2024-09-14T03:38:11.968 INFO:teuthology.orchestra.run.smithi083.stdout:Dependencies resolved. 2024-09-14T03:38:11.969 INFO:teuthology.orchestra.run.smithi083.stdout:Nothing to do. 2024-09-14T03:38:11.969 INFO:teuthology.orchestra.run.smithi083.stdout:Complete! 2024-09-14T03:38:12.028 DEBUG:teuthology.orchestra.run.smithi083:> sudo yum -y install python3-rbd 2024-09-14T03:38:12.597 INFO:teuthology.orchestra.run.smithi083.stdout:Last metadata expiration check: 0:01:08 ago on Sat 14 Sep 2024 03:37:04 AM UTC. 2024-09-14T03:38:12.689 INFO:teuthology.orchestra.run.smithi083.stdout:Package python3-rbd-2:19.1.1-324.gecab7be5.el9.x86_64 is already installed. 2024-09-14T03:38:12.734 INFO:teuthology.orchestra.run.smithi083.stdout:Dependencies resolved. 2024-09-14T03:38:12.735 INFO:teuthology.orchestra.run.smithi083.stdout:Nothing to do. 2024-09-14T03:38:12.735 INFO:teuthology.orchestra.run.smithi083.stdout:Complete! 2024-09-14T03:38:12.806 DEBUG:teuthology.orchestra.run.smithi083:> sudo yum -y install rbd-fuse 2024-09-14T03:38:13.390 INFO:teuthology.orchestra.run.smithi083.stdout:Last metadata expiration check: 0:01:09 ago on Sat 14 Sep 2024 03:37:04 AM UTC. 2024-09-14T03:38:13.528 INFO:teuthology.orchestra.run.smithi083.stdout:Dependencies resolved. 2024-09-14T03:38:13.529 INFO:teuthology.orchestra.run.smithi083.stdout:================================================================================ 2024-09-14T03:38:13.529 INFO:teuthology.orchestra.run.smithi083.stdout: Package Architecture Version Repository Size 2024-09-14T03:38:13.529 INFO:teuthology.orchestra.run.smithi083.stdout:================================================================================ 2024-09-14T03:38:13.529 INFO:teuthology.orchestra.run.smithi083.stdout:Installing: 2024-09-14T03:38:13.529 INFO:teuthology.orchestra.run.smithi083.stdout: rbd-fuse x86_64 2:19.1.1-324.gecab7be5.el9 ceph 85 k 2024-09-14T03:38:13.529 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:38:13.529 INFO:teuthology.orchestra.run.smithi083.stdout:Transaction Summary 2024-09-14T03:38:13.529 INFO:teuthology.orchestra.run.smithi083.stdout:================================================================================ 2024-09-14T03:38:13.529 INFO:teuthology.orchestra.run.smithi083.stdout:Install 1 Package 2024-09-14T03:38:13.529 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:38:13.529 INFO:teuthology.orchestra.run.smithi083.stdout:Total download size: 85 k 2024-09-14T03:38:13.530 INFO:teuthology.orchestra.run.smithi083.stdout:Installed size: 231 k 2024-09-14T03:38:13.530 INFO:teuthology.orchestra.run.smithi083.stdout:Downloading Packages: 2024-09-14T03:38:13.728 INFO:teuthology.orchestra.run.smithi083.stdout:rbd-fuse-19.1.1-324.gecab7be5.el9.x86_64.rpm 429 kB/s | 85 kB 00:00 2024-09-14T03:38:13.729 INFO:teuthology.orchestra.run.smithi083.stdout:-------------------------------------------------------------------------------- 2024-09-14T03:38:13.729 INFO:teuthology.orchestra.run.smithi083.stdout:Total 426 kB/s | 85 kB 00:00 2024-09-14T03:38:13.729 INFO:teuthology.orchestra.run.smithi083.stdout:Running transaction check 2024-09-14T03:38:13.737 INFO:teuthology.orchestra.run.smithi083.stdout:Transaction check succeeded. 2024-09-14T03:38:13.737 INFO:teuthology.orchestra.run.smithi083.stdout:Running transaction test 2024-09-14T03:38:13.793 INFO:teuthology.orchestra.run.smithi083.stdout:Transaction test succeeded. 2024-09-14T03:38:13.793 INFO:teuthology.orchestra.run.smithi083.stdout:Running transaction 2024-09-14T03:38:13.931 INFO:teuthology.orchestra.run.smithi083.stdout: Preparing : 1/1 2024-09-14T03:38:14.024 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : rbd-fuse-2:19.1.1-324.gecab7be5.el9.x86_64 1/1 2024-09-14T03:38:14.413 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: rbd-fuse-2:19.1.1-324.gecab7be5.el9.x86_64 1/1 2024-09-14T03:38:14.671 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : rbd-fuse-2:19.1.1-324.gecab7be5.el9.x86_64 1/1 2024-09-14T03:38:14.671 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:38:14.671 INFO:teuthology.orchestra.run.smithi083.stdout:Installed: 2024-09-14T03:38:14.672 INFO:teuthology.orchestra.run.smithi083.stdout: rbd-fuse-2:19.1.1-324.gecab7be5.el9.x86_64 2024-09-14T03:38:14.672 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:38:14.672 INFO:teuthology.orchestra.run.smithi083.stdout:Complete! 2024-09-14T03:38:14.895 DEBUG:teuthology.orchestra.run.smithi083:> sudo yum -y install rbd-mirror 2024-09-14T03:38:15.462 INFO:teuthology.orchestra.run.smithi083.stdout:Last metadata expiration check: 0:01:11 ago on Sat 14 Sep 2024 03:37:04 AM UTC. 2024-09-14T03:38:15.595 INFO:teuthology.orchestra.run.smithi083.stdout:Dependencies resolved. 2024-09-14T03:38:15.595 INFO:teuthology.orchestra.run.smithi083.stdout:================================================================================ 2024-09-14T03:38:15.596 INFO:teuthology.orchestra.run.smithi083.stdout: Package Arch Version Repo Size 2024-09-14T03:38:15.596 INFO:teuthology.orchestra.run.smithi083.stdout:================================================================================ 2024-09-14T03:38:15.596 INFO:teuthology.orchestra.run.smithi083.stdout:Installing: 2024-09-14T03:38:15.596 INFO:teuthology.orchestra.run.smithi083.stdout: rbd-mirror x86_64 2:19.1.1-324.gecab7be5.el9 ceph 3.1 M 2024-09-14T03:38:15.596 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:38:15.596 INFO:teuthology.orchestra.run.smithi083.stdout:Transaction Summary 2024-09-14T03:38:15.596 INFO:teuthology.orchestra.run.smithi083.stdout:================================================================================ 2024-09-14T03:38:15.596 INFO:teuthology.orchestra.run.smithi083.stdout:Install 1 Package 2024-09-14T03:38:15.596 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:38:15.596 INFO:teuthology.orchestra.run.smithi083.stdout:Total download size: 3.1 M 2024-09-14T03:38:15.596 INFO:teuthology.orchestra.run.smithi083.stdout:Installed size: 13 M 2024-09-14T03:38:15.596 INFO:teuthology.orchestra.run.smithi083.stdout:Downloading Packages: 2024-09-14T03:38:15.941 INFO:teuthology.orchestra.run.smithi083.stdout:rbd-mirror-19.1.1-324.gecab7be5.el9.x86_64.rpm 9.0 MB/s | 3.1 MB 00:00 2024-09-14T03:38:15.942 INFO:teuthology.orchestra.run.smithi083.stdout:-------------------------------------------------------------------------------- 2024-09-14T03:38:15.942 INFO:teuthology.orchestra.run.smithi083.stdout:Total 9.0 MB/s | 3.1 MB 00:00 2024-09-14T03:38:15.942 INFO:teuthology.orchestra.run.smithi083.stdout:Running transaction check 2024-09-14T03:38:15.951 INFO:teuthology.orchestra.run.smithi083.stdout:Transaction check succeeded. 2024-09-14T03:38:15.951 INFO:teuthology.orchestra.run.smithi083.stdout:Running transaction test 2024-09-14T03:38:16.025 INFO:teuthology.orchestra.run.smithi083.stdout:Transaction test succeeded. 2024-09-14T03:38:16.025 INFO:teuthology.orchestra.run.smithi083.stdout:Running transaction 2024-09-14T03:38:16.450 INFO:teuthology.orchestra.run.smithi083.stdout: Preparing : 1/1 2024-09-14T03:38:16.542 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : rbd-mirror-2:19.1.1-324.gecab7be5.el9.x86_64 1/1 2024-09-14T03:38:16.567 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: rbd-mirror-2:19.1.1-324.gecab7be5.el9.x86_64 1/1 2024-09-14T03:38:16.567 INFO:teuthology.orchestra.run.smithi083.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-14T03:38:16.567 INFO:teuthology.orchestra.run.smithi083.stdout:Invalid unit name "ceph-rbd-mirror@*.service" escaped as "ceph-rbd-mirror@\x2a.service". 2024-09-14T03:38:16.567 INFO:teuthology.orchestra.run.smithi083.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-09-14T03:38:16.567 INFO:teuthology.orchestra.run.smithi083.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-09-14T03:38:16.567 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:38:17.596 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : rbd-mirror-2:19.1.1-324.gecab7be5.el9.x86_64 1/1 2024-09-14T03:38:17.596 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:38:17.596 INFO:teuthology.orchestra.run.smithi083.stdout:Installed: 2024-09-14T03:38:17.596 INFO:teuthology.orchestra.run.smithi083.stdout: rbd-mirror-2:19.1.1-324.gecab7be5.el9.x86_64 2024-09-14T03:38:17.596 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:38:17.596 INFO:teuthology.orchestra.run.smithi083.stdout:Complete! 2024-09-14T03:38:17.822 DEBUG:teuthology.orchestra.run.smithi083:> sudo yum -y install rbd-nbd 2024-09-14T03:38:18.397 INFO:teuthology.orchestra.run.smithi083.stdout:Last metadata expiration check: 0:01:14 ago on Sat 14 Sep 2024 03:37:04 AM UTC. 2024-09-14T03:38:18.532 INFO:teuthology.orchestra.run.smithi083.stdout:Dependencies resolved. 2024-09-14T03:38:18.532 INFO:teuthology.orchestra.run.smithi083.stdout:================================================================================ 2024-09-14T03:38:18.532 INFO:teuthology.orchestra.run.smithi083.stdout: Package Architecture Version Repository Size 2024-09-14T03:38:18.532 INFO:teuthology.orchestra.run.smithi083.stdout:================================================================================ 2024-09-14T03:38:18.532 INFO:teuthology.orchestra.run.smithi083.stdout:Installing: 2024-09-14T03:38:18.532 INFO:teuthology.orchestra.run.smithi083.stdout: rbd-nbd x86_64 2:19.1.1-324.gecab7be5.el9 ceph 170 k 2024-09-14T03:38:18.533 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:38:18.533 INFO:teuthology.orchestra.run.smithi083.stdout:Transaction Summary 2024-09-14T03:38:18.533 INFO:teuthology.orchestra.run.smithi083.stdout:================================================================================ 2024-09-14T03:38:18.533 INFO:teuthology.orchestra.run.smithi083.stdout:Install 1 Package 2024-09-14T03:38:18.533 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:38:18.533 INFO:teuthology.orchestra.run.smithi083.stdout:Total download size: 170 k 2024-09-14T03:38:18.533 INFO:teuthology.orchestra.run.smithi083.stdout:Installed size: 494 k 2024-09-14T03:38:18.534 INFO:teuthology.orchestra.run.smithi083.stdout:Downloading Packages: 2024-09-14T03:38:18.736 INFO:teuthology.orchestra.run.smithi083.stdout:rbd-nbd-19.1.1-324.gecab7be5.el9.x86_64.rpm 842 kB/s | 170 kB 00:00 2024-09-14T03:38:18.737 INFO:teuthology.orchestra.run.smithi083.stdout:-------------------------------------------------------------------------------- 2024-09-14T03:38:18.737 INFO:teuthology.orchestra.run.smithi083.stdout:Total 835 kB/s | 170 kB 00:00 2024-09-14T03:38:18.737 INFO:teuthology.orchestra.run.smithi083.stdout:Running transaction check 2024-09-14T03:38:18.744 INFO:teuthology.orchestra.run.smithi083.stdout:Transaction check succeeded. 2024-09-14T03:38:18.745 INFO:teuthology.orchestra.run.smithi083.stdout:Running transaction test 2024-09-14T03:38:18.802 INFO:teuthology.orchestra.run.smithi083.stdout:Transaction test succeeded. 2024-09-14T03:38:18.802 INFO:teuthology.orchestra.run.smithi083.stdout:Running transaction 2024-09-14T03:38:18.946 INFO:teuthology.orchestra.run.smithi083.stdout: Preparing : 1/1 2024-09-14T03:38:19.040 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : rbd-nbd-2:19.1.1-324.gecab7be5.el9.x86_64 1/1 2024-09-14T03:38:19.450 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: rbd-nbd-2:19.1.1-324.gecab7be5.el9.x86_64 1/1 2024-09-14T03:38:19.704 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : rbd-nbd-2:19.1.1-324.gecab7be5.el9.x86_64 1/1 2024-09-14T03:38:19.704 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:38:19.704 INFO:teuthology.orchestra.run.smithi083.stdout:Installed: 2024-09-14T03:38:19.704 INFO:teuthology.orchestra.run.smithi083.stdout: rbd-nbd-2:19.1.1-324.gecab7be5.el9.x86_64 2024-09-14T03:38:19.705 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:38:19.705 INFO:teuthology.orchestra.run.smithi083.stdout:Complete! 2024-09-14T03:38:19.904 DEBUG:teuthology.orchestra.run.smithi083:> sudo yum -y install librados2 2024-09-14T03:38:20.467 INFO:teuthology.orchestra.run.smithi083.stdout:Last metadata expiration check: 0:01:16 ago on Sat 14 Sep 2024 03:37:04 AM UTC. 2024-09-14T03:38:20.557 INFO:teuthology.orchestra.run.smithi083.stdout:Package librados2-2:19.1.1-324.gecab7be5.el9.x86_64 is already installed. 2024-09-14T03:38:20.599 INFO:teuthology.orchestra.run.smithi083.stdout:Dependencies resolved. 2024-09-14T03:38:20.600 INFO:teuthology.orchestra.run.smithi083.stdout:Nothing to do. 2024-09-14T03:38:20.600 INFO:teuthology.orchestra.run.smithi083.stdout:Complete! 2024-09-14T03:38:20.703 DEBUG:teuthology.parallel:result is None 2024-09-14T03:38:20.703 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-14T03:38:20.703 INFO:teuthology.packaging:ref: None 2024-09-14T03:38:20.703 INFO:teuthology.packaging:tag: None 2024-09-14T03:38:20.703 INFO:teuthology.packaging:branch: squid 2024-09-14T03:38:20.703 INFO:teuthology.packaging:sha1: a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T03:38:20.704 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=squid 2024-09-14T03:38:20.934 DEBUG:teuthology.orchestra.run.smithi083:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2024-09-14T03:38:20.955 INFO:teuthology.orchestra.run.smithi083.stdout:19.1.1-324.gecab7be5.el9 2024-09-14T03:38:20.956 INFO:teuthology.packaging:The installed version of ceph is 19.1.1-324.gecab7be5.el9 2024-09-14T03:38:20.956 INFO:teuthology.task.install:The correct ceph version 19.1.1-324.gecab7be5 is installed. 2024-09-14T03:38:20.958 INFO:teuthology.task.install.util:Shipping valgrind.supp... 2024-09-14T03:38:20.959 DEBUG:teuthology.orchestra.run.smithi083:> set -ex 2024-09-14T03:38:20.959 DEBUG:teuthology.orchestra.run.smithi083:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2024-09-14T03:38:21.021 INFO:teuthology.task.install.util:Shipping 'daemon-helper'... 2024-09-14T03:38:21.025 DEBUG:teuthology.orchestra.run.smithi083:> set -ex 2024-09-14T03:38:21.025 DEBUG:teuthology.orchestra.run.smithi083:> sudo dd of=/usr/bin/daemon-helper 2024-09-14T03:38:21.086 DEBUG:teuthology.orchestra.run.smithi083:> sudo chmod a=rx -- /usr/bin/daemon-helper 2024-09-14T03:38:21.151 INFO:teuthology.task.install.util:Shipping 'adjust-ulimits'... 2024-09-14T03:38:21.151 DEBUG:teuthology.orchestra.run.smithi083:> set -ex 2024-09-14T03:38:21.151 DEBUG:teuthology.orchestra.run.smithi083:> sudo dd of=/usr/bin/adjust-ulimits 2024-09-14T03:38:21.216 DEBUG:teuthology.orchestra.run.smithi083:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2024-09-14T03:38:21.282 INFO:teuthology.task.install.util:Shipping 'stdin-killer'... 2024-09-14T03:38:21.282 DEBUG:teuthology.orchestra.run.smithi083:> set -ex 2024-09-14T03:38:21.282 DEBUG:teuthology.orchestra.run.smithi083:> sudo dd of=/usr/bin/stdin-killer 2024-09-14T03:38:21.347 DEBUG:teuthology.orchestra.run.smithi083:> sudo chmod a=rx -- /usr/bin/stdin-killer 2024-09-14T03:38:21.411 INFO:teuthology.run_tasks:Running task print... 2024-09-14T03:38:21.421 INFO:teuthology.task.print:**** done installing squid 2024-09-14T03:38:21.421 INFO:teuthology.run_tasks:Running task ceph... 2024-09-14T03:38:21.537 INFO:tasks.ceph:Making ceph log dir writeable by non-root... 2024-09-14T03:38:21.537 DEBUG:teuthology.orchestra.run.smithi083:> sudo chmod 777 /var/log/ceph 2024-09-14T03:38:21.561 INFO:tasks.ceph:Disabling ceph logrotate... 2024-09-14T03:38:21.561 DEBUG:teuthology.orchestra.run.smithi083:> sudo rm -f -- /etc/logrotate.d/ceph 2024-09-14T03:38:21.626 INFO:tasks.ceph:Creating extra log directories... 2024-09-14T03:38:21.626 DEBUG:teuthology.orchestra.run.smithi083:> sudo install -d -m0777 -- /var/log/ceph/valgrind /var/log/ceph/profiling-logger 2024-09-14T03:38:21.698 INFO:tasks.ceph:Creating ceph cluster ceph... 2024-09-14T03:38:21.698 INFO:tasks.ceph:config {'conf': {'global': {'mon warn on pool no app': False, 'ms bind msgr2': False, 'bluestore warn on legacy statfs': False, 'bluestore warn on no per pool omap': False, 'mon pg warn min per osd': 0}, 'mgr': {'client mount timeout': 30, 'debug client': 20, 'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20, 'mon down mkfs grace': 300, 'mon warn on osd down out interval zero': False}, 'osd': {'bdev async discard': True, 'bdev enable discard': True, 'bluestore allocator': 'bitmap', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'fs': 'xfs', 'mkfs_options': None, 'mount_options': None, 'skip_mgr_daemons': False, 'log_ignorelist': ['overall HEALTH_', '\\(FS_', '\\(MDS_', '\\(OSD_', '\\(MON_DOWN\\)', '\\(CACHE_POOL_', '\\(POOL_', '\\(MGR_DOWN\\)', '\\(PG_', '\\(SMALLER_PGP_NUM\\)', 'Monitor daemon marked osd', 'Behind on trimming', 'Manager daemon', '\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', 'FS_DEGRADED', 'fs.*is degraded', 'filesystem is degraded', 'FS_INLINE_DATA_DEPRECATED', 'FS_WITH_FAILED_MDS', 'MDS_ALL_DOWN', 'filesystem is offline', 'MDS_DAMAGE', 'MDS_DEGRADED', 'MDS_FAILED', 'MDS_INSUFFICIENT_STANDBY', 'insufficient standby MDS daemons available', 'MDS_UP_LESS_THAN_MAX', 'online, but wants', 'filesystem is online with fewer MDS than max_mds', 'POOL_APP_NOT_ENABLED', 'do not have an application enabled', 'overall HEALTH_', 'Replacing daemon', 'deprecated feature inline_data', 'overall HEALTH_', '\\(OSD_DOWN\\)', '\\(OSD_', 'but it is still running', 'is not responding', 'PG_AVAILABILITY', 'PG_DEGRADED', 'Reduced data availability', 'Degraded data redundancy', 'scrub mismatch', 'ScrubResult', 'wrongly marked', '\\(POOL_APP_NOT_ENABLED\\)', '\\(SLOW_OPS\\)', 'overall HEALTH_', '\\(MON_MSGR2_NOT_ENABLED\\)', 'slow request'], 'cpu_profile': set(), 'cluster': 'ceph', 'mon_bind_msgr2': True, 'mon_bind_addrvec': True} 2024-09-14T03:38:21.699 INFO:tasks.ceph:ctx.config {'archive_path': '/home/teuthworker/archive/teuthology-2024-09-14_03:08:15-upgrade-main-distro-default-smithi/7904542', 'branch': 'main', 'description': 'upgrade/cephfs/nofs/{bluestore-bitmap centos_9.stream conf/{client mds mgr mon osd} no-mds-cluster overrides/{ignorelist_health ignorelist_wrongly_marked_down pg-warn pg_health} tasks/{0-from/squid 1-upgrade}}', 'email': 'ceph-qa@ceph.com', 'first_in_suite': False, 'job_id': '7904542', 'kernel': {'kdb': 1, 'sha1': 'distro'}, 'last_in_suite': False, 'machine_type': 'smithi', 'meta': [{'desc': 'install ceph/squid latest\n'}], 'name': 'teuthology-2024-09-14_03:08:15-upgrade-main-distro-default-smithi', 'no_nested_subset': False, 'openstack': [{'volumes': {'count': 4, 'size': 10}}], 'os_type': 'centos', 'os_version': '9.stream', 'overrides': {'admin_socket': {'branch': 'main'}, 'ceph': {'cluster-conf': {'client': {'client mount timeout': 600, 'debug client': 20, 'debug ms': 1, 'rados mon op timeout': 900, 'rados osd op timeout': 900}, 'mds': {'debug mds': 20, 'debug mds balancer': 20, 'debug ms': 2, 'mds debug frag': True, 'mds debug scatterstat': True, 'mds op complaint time': 180, 'mds verify scatter': True, 'osd op complaint time': 180, 'rados mon op timeout': 900, 'rados osd op timeout': 900}, 'mon': {'mon op complaint time': 120, 'mon warn on pool no app': False}, 'osd': {'osd op complaint time': 180}}, 'conf': {'global': {'bluestore warn on legacy statfs': False, 'bluestore warn on no per pool omap': False, 'mon pg warn min per osd': 0}, 'mgr': {'client mount timeout': 30, 'debug client': 20, 'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20, 'mon down mkfs grace': 300, 'mon warn on osd down out interval zero': False}, 'osd': {'bdev async discard': True, 'bdev enable discard': True, 'bluestore allocator': 'bitmap', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', 'FS_DEGRADED', 'fs.*is degraded', 'filesystem is degraded', 'FS_INLINE_DATA_DEPRECATED', 'FS_WITH_FAILED_MDS', 'MDS_ALL_DOWN', 'filesystem is offline', 'MDS_DAMAGE', 'MDS_DEGRADED', 'MDS_FAILED', 'MDS_INSUFFICIENT_STANDBY', 'insufficient standby MDS daemons available', 'MDS_UP_LESS_THAN_MAX', 'online, but wants', 'filesystem is online with fewer MDS than max_mds', 'POOL_APP_NOT_ENABLED', 'do not have an application enabled', 'overall HEALTH_', 'Replacing daemon', 'deprecated feature inline_data', 'overall HEALTH_', '\\(OSD_DOWN\\)', '\\(OSD_', 'but it is still running', 'is not responding', 'PG_AVAILABILITY', 'PG_DEGRADED', 'Reduced data availability', 'Degraded data redundancy', 'scrub mismatch', 'ScrubResult', 'wrongly marked', '\\(POOL_APP_NOT_ENABLED\\)', '\\(SLOW_OPS\\)', 'overall HEALTH_', '\\(MON_MSGR2_NOT_ENABLED\\)', 'slow request'], 'sha1': 'a2782315255a3e7d5f6cdc9eb35785187ff6ddf8'}, 'ceph-deploy': {'bluestore': True, 'conf': {'client': {'log file': '/var/log/ceph/ceph-$name.$pid.log'}, 'mon': {}, 'osd': {'bdev async discard': True, 'bdev enable discard': True, 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'fs': 'xfs'}, 'install': {'ceph': {'flavor': 'default', 'sha1': 'a2782315255a3e7d5f6cdc9eb35785187ff6ddf8'}}, 'selinux': {'allowlist': ['scontext=system_u:system_r:logrotate_t:s0']}, 'thrashosds': {'bdev_inject_crash': 2, 'bdev_inject_crash_probability': 0.5}, 'workunit': {'branch': 'main', 'sha1': '1c941a66c1b420259b8fc5ac19e792dbbf60aded'}}, 'owner': 'scheduled_teuthology@teuthology', 'priority': 850, 'repo': 'https://git.ceph.com/ceph.git', 'roles': [['mon.a', 'mon.b', 'mon.c', 'mgr.x', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3']], 'seed': 8038, 'sha1': 'a2782315255a3e7d5f6cdc9eb35785187ff6ddf8', 'sleep_before_teardown': 0, 'subset': '31/32', 'suite': 'upgrade', 'suite_branch': 'main', 'suite_path': '/home/teuthworker/src/git.ceph.com_ceph_1c941a66c1b420259b8fc5ac19e792dbbf60aded/qa', 'suite_relpath': 'qa', 'suite_repo': 'https://git.ceph.com/ceph.git', 'suite_sha1': '1c941a66c1b420259b8fc5ac19e792dbbf60aded', 'targets': {'smithi083.front.sepia.ceph.com': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBIZoFnW86DoNtGbH4n6H+OCmegCYz4pyfROCf0vMbq7KBlfWRiL2WlsvNlIzYA8hvnHwFYRtAzMd1W6phaUAILc='}, 'tasks': [{'internal.check_packages': None}, {'internal.buildpackages_prep': None}, {'internal.save_config': None}, {'internal.check_lock': None}, {'internal.add_remotes': None}, {'console_log': None}, {'internal.connect': None}, {'internal.push_inventory': None}, {'internal.serialize_remote_roles': None}, {'internal.check_conflict': None}, {'internal.check_ceph_data': None}, {'internal.vm_setup': None}, {'kernel': {'kdb': 1, 'sha1': 'distro'}}, {'internal.base': None}, {'internal.archive_upload': None}, {'internal.archive': None}, {'internal.coredump': None}, {'internal.sudo': None}, {'internal.syslog': None}, {'internal.timer': None}, {'pcp': None}, {'selinux': None}, {'ansible.cephlab': None}, {'clock': None}, {'install': {'branch': 'squid', 'exclude_packages': ['librados3', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'cephadm', 'ceph-volume'], 'extra_packages': ['librados2'], 'flavor': 'default', 'sha1': 'a2782315255a3e7d5f6cdc9eb35785187ff6ddf8'}}, {'print': '**** done installing squid'}, {'ceph': {'conf': {'global': {'mon warn on pool no app': False, 'ms bind msgr2': False, 'bluestore warn on legacy statfs': False, 'bluestore warn on no per pool omap': False, 'mon pg warn min per osd': 0}, 'mgr': {'client mount timeout': 30, 'debug client': 20, 'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20, 'mon down mkfs grace': 300, 'mon warn on osd down out interval zero': False}, 'osd': {'bdev async discard': True, 'bdev enable discard': True, 'bluestore allocator': 'bitmap', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'log-ignorelist': ['overall HEALTH_', '\\(FS_', '\\(MDS_', '\\(OSD_', '\\(MON_DOWN\\)', '\\(CACHE_POOL_', '\\(POOL_', '\\(MGR_DOWN\\)', '\\(PG_', '\\(SMALLER_PGP_NUM\\)', 'Monitor daemon marked osd', 'Behind on trimming', 'Manager daemon', '\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', 'FS_DEGRADED', 'fs.*is degraded', 'filesystem is degraded', 'FS_INLINE_DATA_DEPRECATED', 'FS_WITH_FAILED_MDS', 'MDS_ALL_DOWN', 'filesystem is offline', 'MDS_DAMAGE', 'MDS_DEGRADED', 'MDS_FAILED', 'MDS_INSUFFICIENT_STANDBY', 'insufficient standby MDS daemons available', 'MDS_UP_LESS_THAN_MAX', 'online, but wants', 'filesystem is online with fewer MDS than max_mds', 'POOL_APP_NOT_ENABLED', 'do not have an application enabled', 'overall HEALTH_', 'Replacing daemon', 'deprecated feature inline_data', 'overall HEALTH_', '\\(OSD_DOWN\\)', '\\(OSD_', 'but it is still running', 'is not responding', 'PG_AVAILABILITY', 'PG_DEGRADED', 'Reduced data availability', 'Degraded data redundancy', 'scrub mismatch', 'ScrubResult', 'wrongly marked', '\\(POOL_APP_NOT_ENABLED\\)', '\\(SLOW_OPS\\)', 'overall HEALTH_', '\\(MON_MSGR2_NOT_ENABLED\\)', 'slow request'], 'cluster-conf': {'client': {'client mount timeout': 600, 'debug client': 20, 'debug ms': 1, 'rados mon op timeout': 900, 'rados osd op timeout': 900}, 'mds': {'debug mds': 20, 'debug mds balancer': 20, 'debug ms': 2, 'mds debug frag': True, 'mds debug scatterstat': True, 'mds op complaint time': 180, 'mds verify scatter': True, 'osd op complaint time': 180, 'rados mon op timeout': 900, 'rados osd op timeout': 900}, 'mon': {'mon op complaint time': 120, 'mon warn on pool no app': False}, 'osd': {'osd op complaint time': 180}}, 'flavor': 'default', 'fs': 'xfs', 'sha1': 'a2782315255a3e7d5f6cdc9eb35785187ff6ddf8', 'cluster': 'ceph'}}, {'exec': {'osd.0': ['ceph osd set-require-min-compat-client squid']}}, {'print': '**** done ceph'}, {'print': '*** upgrading, no cephfs present'}, {'exec': {'mon.a': ['ceph fs dump']}}, {'install.upgrade': {'mon.a': None}}, {'print': '**** done install.upgrade'}, {'ceph.restart': {'daemons': ['mon.*', 'mgr.*'], 'mon-health-to-clog': False, 'wait-for-healthy': False}}, {'ceph.healthy': None}, {'ceph.restart': {'daemons': ['osd.*'], 'wait-for-healthy': False, 'wait-for-osds-up': True}}, {'exec': {'mon.a': ['ceph versions', 'ceph osd dump -f json-pretty', 'ceph fs dump', 'ceph osd require-osd-release squid', 'for f in `ceph osd pool ls` ; do ceph osd pool set $f pg_autoscale_mode off ; done']}}, {'ceph.healthy': None}, {'print': '**** done ceph.restart'}], 'teuthology': {'fragments_dropped': [], 'meta': {}, 'postmerge': []}, 'teuthology_branch': 'main', 'teuthology_sha1': '3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a', 'timestamp': '2024-09-14_03:08:15', 'tube': 'smithi', 'user': 'teuthology', 'verbose': False, 'worker_log': '/home/teuthworker/archive/worker_logs/dispatcher.smithi.3401856'} 2024-09-14T03:38:21.699 DEBUG:teuthology.orchestra.run.smithi083:> install -d -m0755 -- /home/ubuntu/cephtest/ceph.data 2024-09-14T03:38:21.747 DEBUG:teuthology.orchestra.run.smithi083:> sudo install -d -m0777 -- /var/run/ceph 2024-09-14T03:38:21.812 DEBUG:teuthology.orchestra.run.smithi083:> set -ex 2024-09-14T03:38:21.812 DEBUG:teuthology.orchestra.run.smithi083:> dd if=/scratch_devs of=/dev/stdout 2024-09-14T03:38:21.868 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-09-14T03:38:21.868 DEBUG:teuthology.orchestra.run.smithi083:> stat /dev/vg_nvme/lv_1 2024-09-14T03:38:21.923 INFO:teuthology.orchestra.run.smithi083.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-09-14T03:38:21.924 INFO:teuthology.orchestra.run.smithi083.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-14T03:38:21.924 INFO:teuthology.orchestra.run.smithi083.stdout:Device: 5h/5d Inode: 668 Links: 1 2024-09-14T03:38:21.924 INFO:teuthology.orchestra.run.smithi083.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-14T03:38:21.924 INFO:teuthology.orchestra.run.smithi083.stdout:Context: system_u:object_r:device_t:s0 2024-09-14T03:38:21.924 INFO:teuthology.orchestra.run.smithi083.stdout:Access: 2024-09-14 03:38:16.813925601 +0000 2024-09-14T03:38:21.924 INFO:teuthology.orchestra.run.smithi083.stdout:Modify: 2024-09-14 03:36:23.807467794 +0000 2024-09-14T03:38:21.924 INFO:teuthology.orchestra.run.smithi083.stdout:Change: 2024-09-14 03:36:23.807467794 +0000 2024-09-14T03:38:21.924 INFO:teuthology.orchestra.run.smithi083.stdout: Birth: 2024-09-14 03:36:23.807467794 +0000 2024-09-14T03:38:21.924 DEBUG:teuthology.orchestra.run.smithi083:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-09-14T03:38:21.988 INFO:teuthology.orchestra.run.smithi083.stderr:1+0 records in 2024-09-14T03:38:21.988 INFO:teuthology.orchestra.run.smithi083.stderr:1+0 records out 2024-09-14T03:38:21.988 INFO:teuthology.orchestra.run.smithi083.stderr:512 bytes copied, 0.000203965 s, 2.5 MB/s 2024-09-14T03:38:21.989 DEBUG:teuthology.orchestra.run.smithi083:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-09-14T03:38:22.045 DEBUG:teuthology.orchestra.run.smithi083:> stat /dev/vg_nvme/lv_2 2024-09-14T03:38:22.100 INFO:teuthology.orchestra.run.smithi083.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-09-14T03:38:22.100 INFO:teuthology.orchestra.run.smithi083.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-14T03:38:22.101 INFO:teuthology.orchestra.run.smithi083.stdout:Device: 5h/5d Inode: 685 Links: 1 2024-09-14T03:38:22.101 INFO:teuthology.orchestra.run.smithi083.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-14T03:38:22.101 INFO:teuthology.orchestra.run.smithi083.stdout:Context: system_u:object_r:device_t:s0 2024-09-14T03:38:22.101 INFO:teuthology.orchestra.run.smithi083.stdout:Access: 2024-09-14 03:38:16.814925577 +0000 2024-09-14T03:38:22.101 INFO:teuthology.orchestra.run.smithi083.stdout:Modify: 2024-09-14 03:36:24.135459555 +0000 2024-09-14T03:38:22.101 INFO:teuthology.orchestra.run.smithi083.stdout:Change: 2024-09-14 03:36:24.135459555 +0000 2024-09-14T03:38:22.101 INFO:teuthology.orchestra.run.smithi083.stdout: Birth: 2024-09-14 03:36:24.135459555 +0000 2024-09-14T03:38:22.101 DEBUG:teuthology.orchestra.run.smithi083:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-09-14T03:38:22.176 INFO:teuthology.orchestra.run.smithi083.stderr:1+0 records in 2024-09-14T03:38:22.176 INFO:teuthology.orchestra.run.smithi083.stderr:1+0 records out 2024-09-14T03:38:22.176 INFO:teuthology.orchestra.run.smithi083.stderr:512 bytes copied, 0.000220941 s, 2.3 MB/s 2024-09-14T03:38:22.177 DEBUG:teuthology.orchestra.run.smithi083:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-09-14T03:38:22.237 DEBUG:teuthology.orchestra.run.smithi083:> stat /dev/vg_nvme/lv_3 2024-09-14T03:38:22.293 INFO:teuthology.orchestra.run.smithi083.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-09-14T03:38:22.293 INFO:teuthology.orchestra.run.smithi083.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-14T03:38:22.293 INFO:teuthology.orchestra.run.smithi083.stdout:Device: 5h/5d Inode: 705 Links: 1 2024-09-14T03:38:22.293 INFO:teuthology.orchestra.run.smithi083.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-14T03:38:22.293 INFO:teuthology.orchestra.run.smithi083.stdout:Context: system_u:object_r:device_t:s0 2024-09-14T03:38:22.293 INFO:teuthology.orchestra.run.smithi083.stdout:Access: 2024-09-14 03:38:16.814925577 +0000 2024-09-14T03:38:22.293 INFO:teuthology.orchestra.run.smithi083.stdout:Modify: 2024-09-14 03:36:24.470451142 +0000 2024-09-14T03:38:22.296 INFO:teuthology.orchestra.run.smithi083.stdout:Change: 2024-09-14 03:36:24.470451142 +0000 2024-09-14T03:38:22.296 INFO:teuthology.orchestra.run.smithi083.stdout: Birth: 2024-09-14 03:36:24.470451142 +0000 2024-09-14T03:38:22.296 DEBUG:teuthology.orchestra.run.smithi083:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-09-14T03:38:22.356 INFO:teuthology.orchestra.run.smithi083.stderr:1+0 records in 2024-09-14T03:38:22.357 INFO:teuthology.orchestra.run.smithi083.stderr:1+0 records out 2024-09-14T03:38:22.357 INFO:teuthology.orchestra.run.smithi083.stderr:512 bytes copied, 0.000195212 s, 2.6 MB/s 2024-09-14T03:38:22.358 DEBUG:teuthology.orchestra.run.smithi083:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-09-14T03:38:22.414 DEBUG:teuthology.orchestra.run.smithi083:> stat /dev/vg_nvme/lv_4 2024-09-14T03:38:22.469 INFO:teuthology.orchestra.run.smithi083.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-09-14T03:38:22.469 INFO:teuthology.orchestra.run.smithi083.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-14T03:38:22.469 INFO:teuthology.orchestra.run.smithi083.stdout:Device: 5h/5d Inode: 723 Links: 1 2024-09-14T03:38:22.469 INFO:teuthology.orchestra.run.smithi083.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-14T03:38:22.470 INFO:teuthology.orchestra.run.smithi083.stdout:Context: system_u:object_r:device_t:s0 2024-09-14T03:38:22.470 INFO:teuthology.orchestra.run.smithi083.stdout:Access: 2024-09-14 03:38:16.814925577 +0000 2024-09-14T03:38:22.470 INFO:teuthology.orchestra.run.smithi083.stdout:Modify: 2024-09-14 03:36:24.799442879 +0000 2024-09-14T03:38:22.470 INFO:teuthology.orchestra.run.smithi083.stdout:Change: 2024-09-14 03:36:24.799442879 +0000 2024-09-14T03:38:22.470 INFO:teuthology.orchestra.run.smithi083.stdout: Birth: 2024-09-14 03:36:24.799442879 +0000 2024-09-14T03:38:22.470 DEBUG:teuthology.orchestra.run.smithi083:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-09-14T03:38:22.532 INFO:teuthology.orchestra.run.smithi083.stderr:1+0 records in 2024-09-14T03:38:22.532 INFO:teuthology.orchestra.run.smithi083.stderr:1+0 records out 2024-09-14T03:38:22.533 INFO:teuthology.orchestra.run.smithi083.stderr:512 bytes copied, 0.000200255 s, 2.6 MB/s 2024-09-14T03:38:22.534 DEBUG:teuthology.orchestra.run.smithi083:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-09-14T03:38:22.590 INFO:tasks.ceph:osd dev map: {'osd.0': '/dev/vg_nvme/lv_1', 'osd.1': '/dev/vg_nvme/lv_2', 'osd.2': '/dev/vg_nvme/lv_3', 'osd.3': '/dev/vg_nvme/lv_4'} 2024-09-14T03:38:22.590 INFO:tasks.ceph:remote_to_roles_to_devs: {Remote(name='ubuntu@smithi083.front.sepia.ceph.com'): {'osd.0': '/dev/vg_nvme/lv_1', 'osd.1': '/dev/vg_nvme/lv_2', 'osd.2': '/dev/vg_nvme/lv_3', 'osd.3': '/dev/vg_nvme/lv_4'}} 2024-09-14T03:38:22.591 INFO:tasks.ceph:Generating config... 2024-09-14T03:38:22.592 INFO:tasks.ceph:[global] mon warn on pool no app = False 2024-09-14T03:38:22.592 INFO:tasks.ceph:[global] ms bind msgr2 = False 2024-09-14T03:38:22.592 INFO:tasks.ceph:[global] bluestore warn on legacy statfs = False 2024-09-14T03:38:22.592 INFO:tasks.ceph:[global] bluestore warn on no per pool omap = False 2024-09-14T03:38:22.592 INFO:tasks.ceph:[global] mon pg warn min per osd = 0 2024-09-14T03:38:22.592 INFO:tasks.ceph:[mgr] client mount timeout = 30 2024-09-14T03:38:22.592 INFO:tasks.ceph:[mgr] debug client = 20 2024-09-14T03:38:22.592 INFO:tasks.ceph:[mgr] debug mgr = 20 2024-09-14T03:38:22.592 INFO:tasks.ceph:[mgr] debug ms = 1 2024-09-14T03:38:22.592 INFO:tasks.ceph:[mon] debug mon = 20 2024-09-14T03:38:22.593 INFO:tasks.ceph:[mon] debug ms = 1 2024-09-14T03:38:22.593 INFO:tasks.ceph:[mon] debug paxos = 20 2024-09-14T03:38:22.593 INFO:tasks.ceph:[mon] mon down mkfs grace = 300 2024-09-14T03:38:22.593 INFO:tasks.ceph:[mon] mon warn on osd down out interval zero = False 2024-09-14T03:38:22.593 INFO:tasks.ceph:[osd] bdev async discard = True 2024-09-14T03:38:22.593 INFO:tasks.ceph:[osd] bdev enable discard = True 2024-09-14T03:38:22.593 INFO:tasks.ceph:[osd] bluestore allocator = bitmap 2024-09-14T03:38:22.593 INFO:tasks.ceph:[osd] bluestore block size = 96636764160 2024-09-14T03:38:22.593 INFO:tasks.ceph:[osd] bluestore fsck on mount = True 2024-09-14T03:38:22.593 INFO:tasks.ceph:[osd] debug bluefs = 1/20 2024-09-14T03:38:22.593 INFO:tasks.ceph:[osd] debug bluestore = 1/20 2024-09-14T03:38:22.593 INFO:tasks.ceph:[osd] debug ms = 1 2024-09-14T03:38:22.593 INFO:tasks.ceph:[osd] debug osd = 20 2024-09-14T03:38:22.593 INFO:tasks.ceph:[osd] debug rocksdb = 4/10 2024-09-14T03:38:22.594 INFO:tasks.ceph:[osd] mon osd backfillfull_ratio = 0.85 2024-09-14T03:38:22.594 INFO:tasks.ceph:[osd] mon osd full ratio = 0.9 2024-09-14T03:38:22.594 INFO:tasks.ceph:[osd] mon osd nearfull ratio = 0.8 2024-09-14T03:38:22.594 INFO:tasks.ceph:[osd] osd failsafe full ratio = 0.95 2024-09-14T03:38:22.594 INFO:tasks.ceph:[osd] osd objectstore = bluestore 2024-09-14T03:38:22.594 INFO:tasks.ceph:Setting up mon.a... 2024-09-14T03:38:22.594 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring /etc/ceph/ceph.keyring 2024-09-14T03:38:22.679 INFO:teuthology.orchestra.run.smithi083.stdout:creating /etc/ceph/ceph.keyring 2024-09-14T03:38:22.682 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=mon. /etc/ceph/ceph.keyring 2024-09-14T03:38:22.725 DEBUG:teuthology.orchestra.run.smithi083:> sudo chmod 0644 /etc/ceph/ceph.keyring 2024-09-14T03:38:22.790 DEBUG:tasks.ceph:Ceph mon addresses: [('mon.a', '172.21.15.83'), ('mon.b', '[v2:172.21.15.83:3301,v1:172.21.15.83:6790]'), ('mon.c', '[v2:172.21.15.83:3302,v1:172.21.15.83:6791]')] 2024-09-14T03:38:22.790 DEBUG:tasks.ceph:writing out conf {'global': {'chdir': '', 'pid file': '/var/run/ceph/$cluster-$name.pid', 'auth supported': 'cephx', 'filestore xattr use omap': 'true', 'mon clock drift allowed': '1.000', 'osd crush chooseleaf type': '0', 'auth debug': 'true', 'ms die on old message': 'true', 'ms die on bug': 'true', 'mon max pg per osd': '10000', 'mon pg warn max object skew': '0', 'osd_pool_default_pg_autoscale_mode': 'off', 'osd pool default size': '2', 'mon osd allow primary affinity': 'true', 'mon osd allow pg remap': 'true', 'mon warn on legacy crush tunables': 'false', 'mon warn on crush straw calc version zero': 'false', 'mon warn on no sortbitwise': 'false', 'mon warn on osd down out interval zero': 'false', 'mon warn on too few osds': 'false', 'mon_warn_on_pool_pg_num_not_power_of_two': 'false', 'mon_warn_on_pool_no_redundancy': 'false', 'mon_allow_pool_size_one': 'true', 'osd pool default erasure code profile': 'plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd', 'osd default data pool replay window': '5', 'mon allow pool delete': 'true', 'mon cluster log file level': 'debug', 'debug asserts on shutdown': 'true', 'mon health detail to clog': 'false', 'mon host': '172.21.15.83,[v2:172.21.15.83:3301,v1:172.21.15.83:6790],[v2:172.21.15.83:3302,v1:172.21.15.83:6791]', 'mon warn on pool no app': False, 'ms bind msgr2': False, 'bluestore warn on legacy statfs': False, 'bluestore warn on no per pool omap': False, 'mon pg warn min per osd': 0}, 'osd': {'osd journal size': '100', 'osd scrub load threshold': '5.0', 'osd scrub max interval': '600', 'osd mclock profile': 'high_recovery_ops', 'osd recover clone overlap': 'true', 'osd recovery max chunk': '1048576', 'osd debug shutdown': 'true', 'osd debug op order': 'true', 'osd debug verify stray on activate': 'true', 'osd debug trim objects': 'true', 'osd open classes on start': 'true', 'osd debug pg log writeout': 'true', 'osd deep scrub update digest min age': '30', 'osd map max advance': '10', 'journal zero on create': 'true', 'filestore ondisk finisher threads': '3', 'filestore apply finisher threads': '3', 'bdev debug aio': 'true', 'osd debug misdirected ops': 'true', 'bdev async discard': True, 'bdev enable discard': True, 'bluestore allocator': 'bitmap', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}, 'mgr': {'debug ms': 1, 'debug mgr': 20, 'debug mon': '20', 'debug auth': '20', 'mon reweight min pgs per osd': '4', 'mon reweight min bytes per osd': '10', 'mgr/telemetry/nag': 'false', 'client mount timeout': 30, 'debug client': 20}, 'mon': {'debug ms': 1, 'debug mon': 20, 'debug paxos': 20, 'debug auth': '20', 'mon data avail warn': '5', 'mon mgr mkfs grace': '240', 'mon reweight min pgs per osd': '4', 'mon osd reporter subtree level': 'osd', 'mon osd prime pg temp': 'true', 'mon reweight min bytes per osd': '10', 'auth mon ticket ttl': '660', 'auth service ticket ttl': '240', 'mon_warn_on_insecure_global_id_reclaim': 'false', 'mon_warn_on_insecure_global_id_reclaim_allowed': 'false', 'mon_down_mkfs_grace': '2m', 'mon_warn_on_filestore_osds': 'false', 'mon down mkfs grace': 300, 'mon warn on osd down out interval zero': False}, 'client': {'rgw cache enabled': 'true', 'rgw enable ops log': 'true', 'rgw enable usage log': 'true', 'log file': '/var/log/ceph/$cluster-$name.$pid.log', 'admin socket': '/var/run/ceph/$cluster-$name.$pid.asok'}, 'mon.a': {}, 'mon.b': {}, 'mon.c': {}} 2024-09-14T03:38:22.791 DEBUG:teuthology.orchestra.run.smithi083:> set -ex 2024-09-14T03:38:22.791 DEBUG:teuthology.orchestra.run.smithi083:> dd of=/home/ubuntu/cephtest/ceph.tmp.conf 2024-09-14T03:38:22.845 DEBUG:teuthology.orchestra.run.smithi083:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage monmaptool -c /home/ubuntu/cephtest/ceph.tmp.conf --create --clobber --enable-all-features --add a 172.21.15.83 --addv b '[v2:172.21.15.83:3301,v1:172.21.15.83:6790]' --addv c '[v2:172.21.15.83:3302,v1:172.21.15.83:6791]' --print /home/ubuntu/cephtest/ceph.monmap 2024-09-14T03:38:22.931 INFO:teuthology.orchestra.run.smithi083.stdout:monmaptool: monmap file /home/ubuntu/cephtest/ceph.monmap 2024-09-14T03:38:22.932 INFO:teuthology.orchestra.run.smithi083.stdout:monmaptool: generated fsid 765298ef-df4c-4f28-af02-b45f60264ff9 2024-09-14T03:38:22.932 INFO:teuthology.orchestra.run.smithi083.stdout:setting min_mon_release = quincy 2024-09-14T03:38:22.932 INFO:teuthology.orchestra.run.smithi083.stdout:epoch 0 2024-09-14T03:38:22.932 INFO:teuthology.orchestra.run.smithi083.stdout:fsid 765298ef-df4c-4f28-af02-b45f60264ff9 2024-09-14T03:38:22.932 INFO:teuthology.orchestra.run.smithi083.stdout:last_changed 2024-09-14T03:38:22.931060+0000 2024-09-14T03:38:22.932 INFO:teuthology.orchestra.run.smithi083.stdout:created 2024-09-14T03:38:22.931060+0000 2024-09-14T03:38:22.932 INFO:teuthology.orchestra.run.smithi083.stdout:min_mon_release 17 (quincy) 2024-09-14T03:38:22.932 INFO:teuthology.orchestra.run.smithi083.stdout:election_strategy: 1 2024-09-14T03:38:22.932 INFO:teuthology.orchestra.run.smithi083.stdout:0: [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] mon.a 2024-09-14T03:38:22.932 INFO:teuthology.orchestra.run.smithi083.stdout:1: [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] mon.b 2024-09-14T03:38:22.932 INFO:teuthology.orchestra.run.smithi083.stdout:2: [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] mon.c 2024-09-14T03:38:22.932 INFO:teuthology.orchestra.run.smithi083.stdout:monmaptool: writing epoch 0 to /home/ubuntu/cephtest/ceph.monmap (3 monitors) 2024-09-14T03:38:22.933 DEBUG:teuthology.orchestra.run.smithi083:> rm -- /home/ubuntu/cephtest/ceph.tmp.conf 2024-09-14T03:38:22.948 INFO:tasks.ceph:Writing /etc/ceph/ceph.conf for FSID 765298ef-df4c-4f28-af02-b45f60264ff9... 2024-09-14T03:38:22.949 DEBUG:teuthology.orchestra.run.smithi083:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/ceph.conf && sudo chmod 0644 /etc/ceph/ceph.conf > /dev/null 2024-09-14T03:38:23.034 INFO:teuthology.orchestra.run.smithi083.stdout:[global] 2024-09-14T03:38:23.034 INFO:teuthology.orchestra.run.smithi083.stdout: chdir = "" 2024-09-14T03:38:23.035 INFO:teuthology.orchestra.run.smithi083.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2024-09-14T03:38:23.035 INFO:teuthology.orchestra.run.smithi083.stdout: auth supported = cephx 2024-09-14T03:38:23.035 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:38:23.035 INFO:teuthology.orchestra.run.smithi083.stdout: filestore xattr use omap = true 2024-09-14T03:38:23.035 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:38:23.035 INFO:teuthology.orchestra.run.smithi083.stdout: mon clock drift allowed = 1.000 2024-09-14T03:38:23.035 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:38:23.035 INFO:teuthology.orchestra.run.smithi083.stdout: osd crush chooseleaf type = 0 2024-09-14T03:38:23.035 INFO:teuthology.orchestra.run.smithi083.stdout: auth debug = true 2024-09-14T03:38:23.035 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:38:23.035 INFO:teuthology.orchestra.run.smithi083.stdout: ms die on old message = true 2024-09-14T03:38:23.035 INFO:teuthology.orchestra.run.smithi083.stdout: ms die on bug = true 2024-09-14T03:38:23.035 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:38:23.036 INFO:teuthology.orchestra.run.smithi083.stdout: mon max pg per osd = 10000 # >= luminous 2024-09-14T03:38:23.036 INFO:teuthology.orchestra.run.smithi083.stdout: mon pg warn max object skew = 0 2024-09-14T03:38:23.036 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:38:23.036 INFO:teuthology.orchestra.run.smithi083.stdout: # disable pg_autoscaler by default for new pools 2024-09-14T03:38:23.036 INFO:teuthology.orchestra.run.smithi083.stdout: osd_pool_default_pg_autoscale_mode = off 2024-09-14T03:38:23.036 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:38:23.036 INFO:teuthology.orchestra.run.smithi083.stdout: osd pool default size = 2 2024-09-14T03:38:23.036 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:38:23.036 INFO:teuthology.orchestra.run.smithi083.stdout: mon osd allow primary affinity = true 2024-09-14T03:38:23.036 INFO:teuthology.orchestra.run.smithi083.stdout: mon osd allow pg remap = true 2024-09-14T03:38:23.036 INFO:teuthology.orchestra.run.smithi083.stdout: mon warn on legacy crush tunables = false 2024-09-14T03:38:23.036 INFO:teuthology.orchestra.run.smithi083.stdout: mon warn on crush straw calc version zero = false 2024-09-14T03:38:23.036 INFO:teuthology.orchestra.run.smithi083.stdout: mon warn on no sortbitwise = false 2024-09-14T03:38:23.036 INFO:teuthology.orchestra.run.smithi083.stdout: mon warn on osd down out interval zero = false 2024-09-14T03:38:23.037 INFO:teuthology.orchestra.run.smithi083.stdout: mon warn on too few osds = false 2024-09-14T03:38:23.037 INFO:teuthology.orchestra.run.smithi083.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2024-09-14T03:38:23.037 INFO:teuthology.orchestra.run.smithi083.stdout: mon_warn_on_pool_no_redundancy = false 2024-09-14T03:38:23.037 INFO:teuthology.orchestra.run.smithi083.stdout: mon_allow_pool_size_one = true 2024-09-14T03:38:23.037 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:38:23.037 INFO:teuthology.orchestra.run.smithi083.stdout: osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2024-09-14T03:38:23.037 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:38:23.037 INFO:teuthology.orchestra.run.smithi083.stdout: osd default data pool replay window = 5 2024-09-14T03:38:23.037 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:38:23.037 INFO:teuthology.orchestra.run.smithi083.stdout: mon allow pool delete = true 2024-09-14T03:38:23.037 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:38:23.037 INFO:teuthology.orchestra.run.smithi083.stdout: mon cluster log file level = debug 2024-09-14T03:38:23.037 INFO:teuthology.orchestra.run.smithi083.stdout: debug asserts on shutdown = true 2024-09-14T03:38:23.037 INFO:teuthology.orchestra.run.smithi083.stdout: mon health detail to clog = false 2024-09-14T03:38:23.038 INFO:teuthology.orchestra.run.smithi083.stdout: mon host = "172.21.15.83,[v2:172.21.15.83:3301,v1:172.21.15.83:6790],[v2:172.21.15.83:3302,v1:172.21.15.83:6791]" 2024-09-14T03:38:23.038 INFO:teuthology.orchestra.run.smithi083.stdout: mon warn on pool no app = False 2024-09-14T03:38:23.038 INFO:teuthology.orchestra.run.smithi083.stdout: ms bind msgr2 = False 2024-09-14T03:38:23.038 INFO:teuthology.orchestra.run.smithi083.stdout: bluestore warn on legacy statfs = False 2024-09-14T03:38:23.038 INFO:teuthology.orchestra.run.smithi083.stdout: bluestore warn on no per pool omap = False 2024-09-14T03:38:23.038 INFO:teuthology.orchestra.run.smithi083.stdout: mon pg warn min per osd = 0 2024-09-14T03:38:23.038 INFO:teuthology.orchestra.run.smithi083.stdout: fsid = 765298ef-df4c-4f28-af02-b45f60264ff9 2024-09-14T03:38:23.038 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:38:23.038 INFO:teuthology.orchestra.run.smithi083.stdout:[osd] 2024-09-14T03:38:23.038 INFO:teuthology.orchestra.run.smithi083.stdout: osd journal size = 100 2024-09-14T03:38:23.038 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:38:23.038 INFO:teuthology.orchestra.run.smithi083.stdout: osd scrub load threshold = 5.0 2024-09-14T03:38:23.038 INFO:teuthology.orchestra.run.smithi083.stdout: osd scrub max interval = 600 2024-09-14T03:38:23.038 INFO:teuthology.orchestra.run.smithi083.stdout: osd mclock profile = high_recovery_ops 2024-09-14T03:38:23.039 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:38:23.040 INFO:teuthology.orchestra.run.smithi083.stdout: osd recover clone overlap = true 2024-09-14T03:38:23.040 INFO:teuthology.orchestra.run.smithi083.stdout: osd recovery max chunk = 1048576 2024-09-14T03:38:23.040 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:38:23.040 INFO:teuthology.orchestra.run.smithi083.stdout: osd debug shutdown = true 2024-09-14T03:38:23.040 INFO:teuthology.orchestra.run.smithi083.stdout: osd debug op order = true 2024-09-14T03:38:23.040 INFO:teuthology.orchestra.run.smithi083.stdout: osd debug verify stray on activate = true 2024-09-14T03:38:23.040 INFO:teuthology.orchestra.run.smithi083.stdout: osd debug trim objects = true 2024-09-14T03:38:23.040 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:38:23.040 INFO:teuthology.orchestra.run.smithi083.stdout: osd open classes on start = true 2024-09-14T03:38:23.040 INFO:teuthology.orchestra.run.smithi083.stdout: osd debug pg log writeout = true 2024-09-14T03:38:23.040 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:38:23.040 INFO:teuthology.orchestra.run.smithi083.stdout: osd deep scrub update digest min age = 30 2024-09-14T03:38:23.040 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:38:23.044 INFO:teuthology.orchestra.run.smithi083.stdout: osd map max advance = 10 2024-09-14T03:38:23.044 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:38:23.044 INFO:teuthology.orchestra.run.smithi083.stdout: journal zero on create = true 2024-09-14T03:38:23.044 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:38:23.045 INFO:teuthology.orchestra.run.smithi083.stdout: filestore ondisk finisher threads = 3 2024-09-14T03:38:23.045 INFO:teuthology.orchestra.run.smithi083.stdout: filestore apply finisher threads = 3 2024-09-14T03:38:23.045 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:38:23.045 INFO:teuthology.orchestra.run.smithi083.stdout: bdev debug aio = true 2024-09-14T03:38:23.045 INFO:teuthology.orchestra.run.smithi083.stdout: osd debug misdirected ops = true 2024-09-14T03:38:23.045 INFO:teuthology.orchestra.run.smithi083.stdout: bdev async discard = True 2024-09-14T03:38:23.045 INFO:teuthology.orchestra.run.smithi083.stdout: bdev enable discard = True 2024-09-14T03:38:23.045 INFO:teuthology.orchestra.run.smithi083.stdout: bluestore allocator = bitmap 2024-09-14T03:38:23.045 INFO:teuthology.orchestra.run.smithi083.stdout: bluestore block size = 96636764160 2024-09-14T03:38:23.045 INFO:teuthology.orchestra.run.smithi083.stdout: bluestore fsck on mount = True 2024-09-14T03:38:23.045 INFO:teuthology.orchestra.run.smithi083.stdout: debug bluefs = 1/20 2024-09-14T03:38:23.045 INFO:teuthology.orchestra.run.smithi083.stdout: debug bluestore = 1/20 2024-09-14T03:38:23.045 INFO:teuthology.orchestra.run.smithi083.stdout: debug ms = 1 2024-09-14T03:38:23.046 INFO:teuthology.orchestra.run.smithi083.stdout: debug osd = 20 2024-09-14T03:38:23.046 INFO:teuthology.orchestra.run.smithi083.stdout: debug rocksdb = 4/10 2024-09-14T03:38:23.046 INFO:teuthology.orchestra.run.smithi083.stdout: mon osd backfillfull_ratio = 0.85 2024-09-14T03:38:23.046 INFO:teuthology.orchestra.run.smithi083.stdout: mon osd full ratio = 0.9 2024-09-14T03:38:23.046 INFO:teuthology.orchestra.run.smithi083.stdout: mon osd nearfull ratio = 0.8 2024-09-14T03:38:23.046 INFO:teuthology.orchestra.run.smithi083.stdout: osd failsafe full ratio = 0.95 2024-09-14T03:38:23.046 INFO:teuthology.orchestra.run.smithi083.stdout: osd objectstore = bluestore 2024-09-14T03:38:23.046 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:38:23.046 INFO:teuthology.orchestra.run.smithi083.stdout:[mgr] 2024-09-14T03:38:23.046 INFO:teuthology.orchestra.run.smithi083.stdout: debug ms = 1 2024-09-14T03:38:23.046 INFO:teuthology.orchestra.run.smithi083.stdout: debug mgr = 20 2024-09-14T03:38:23.046 INFO:teuthology.orchestra.run.smithi083.stdout: debug mon = 20 2024-09-14T03:38:23.046 INFO:teuthology.orchestra.run.smithi083.stdout: debug auth = 20 2024-09-14T03:38:23.046 INFO:teuthology.orchestra.run.smithi083.stdout: mon reweight min pgs per osd = 4 2024-09-14T03:38:23.047 INFO:teuthology.orchestra.run.smithi083.stdout: mon reweight min bytes per osd = 10 2024-09-14T03:38:23.047 INFO:teuthology.orchestra.run.smithi083.stdout: mgr/telemetry/nag = false 2024-09-14T03:38:23.047 INFO:teuthology.orchestra.run.smithi083.stdout: client mount timeout = 30 2024-09-14T03:38:23.047 INFO:teuthology.orchestra.run.smithi083.stdout: debug client = 20 2024-09-14T03:38:23.047 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:38:23.047 INFO:teuthology.orchestra.run.smithi083.stdout:[mon] 2024-09-14T03:38:23.047 INFO:teuthology.orchestra.run.smithi083.stdout: debug ms = 1 2024-09-14T03:38:23.047 INFO:teuthology.orchestra.run.smithi083.stdout: debug mon = 20 2024-09-14T03:38:23.047 INFO:teuthology.orchestra.run.smithi083.stdout: debug paxos = 20 2024-09-14T03:38:23.047 INFO:teuthology.orchestra.run.smithi083.stdout: debug auth = 20 2024-09-14T03:38:23.047 INFO:teuthology.orchestra.run.smithi083.stdout: mon data avail warn = 5 2024-09-14T03:38:23.047 INFO:teuthology.orchestra.run.smithi083.stdout: mon mgr mkfs grace = 240 2024-09-14T03:38:23.047 INFO:teuthology.orchestra.run.smithi083.stdout: mon reweight min pgs per osd = 4 2024-09-14T03:38:23.047 INFO:teuthology.orchestra.run.smithi083.stdout: mon osd reporter subtree level = osd 2024-09-14T03:38:23.048 INFO:teuthology.orchestra.run.smithi083.stdout: mon osd prime pg temp = true 2024-09-14T03:38:23.048 INFO:teuthology.orchestra.run.smithi083.stdout: mon reweight min bytes per osd = 10 2024-09-14T03:38:23.048 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:38:23.048 INFO:teuthology.orchestra.run.smithi083.stdout: # rotate auth tickets quickly to exercise renewal paths 2024-09-14T03:38:23.048 INFO:teuthology.orchestra.run.smithi083.stdout: auth mon ticket ttl = 660 # 11m 2024-09-14T03:38:23.048 INFO:teuthology.orchestra.run.smithi083.stdout: auth service ticket ttl = 240 # 4m 2024-09-14T03:38:23.048 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:38:23.048 INFO:teuthology.orchestra.run.smithi083.stdout: # don't complain about insecure global_id in the test suite 2024-09-14T03:38:23.048 INFO:teuthology.orchestra.run.smithi083.stdout: mon_warn_on_insecure_global_id_reclaim = false 2024-09-14T03:38:23.048 INFO:teuthology.orchestra.run.smithi083.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2024-09-14T03:38:23.048 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:38:23.048 INFO:teuthology.orchestra.run.smithi083.stdout: # 1m isn't quite enough 2024-09-14T03:38:23.048 INFO:teuthology.orchestra.run.smithi083.stdout: mon_down_mkfs_grace = 2m 2024-09-14T03:38:23.049 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:38:23.049 INFO:teuthology.orchestra.run.smithi083.stdout: mon_warn_on_filestore_osds = false 2024-09-14T03:38:23.049 INFO:teuthology.orchestra.run.smithi083.stdout: mon down mkfs grace = 300 2024-09-14T03:38:23.049 INFO:teuthology.orchestra.run.smithi083.stdout: mon warn on osd down out interval zero = False 2024-09-14T03:38:23.049 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:38:23.049 INFO:teuthology.orchestra.run.smithi083.stdout:[client] 2024-09-14T03:38:23.049 INFO:teuthology.orchestra.run.smithi083.stdout: rgw cache enabled = true 2024-09-14T03:38:23.049 INFO:teuthology.orchestra.run.smithi083.stdout: rgw enable ops log = true 2024-09-14T03:38:23.049 INFO:teuthology.orchestra.run.smithi083.stdout: rgw enable usage log = true 2024-09-14T03:38:23.049 INFO:teuthology.orchestra.run.smithi083.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2024-09-14T03:38:23.049 INFO:teuthology.orchestra.run.smithi083.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2024-09-14T03:38:23.049 INFO:teuthology.orchestra.run.smithi083.stdout:[mon.a] 2024-09-14T03:38:23.049 INFO:teuthology.orchestra.run.smithi083.stdout:[mon.b] 2024-09-14T03:38:23.049 INFO:teuthology.orchestra.run.smithi083.stdout:[mon.c] 2024-09-14T03:38:23.050 INFO:tasks.ceph:Creating admin key on mon.a... 2024-09-14T03:38:23.051 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *' /etc/ceph/ceph.keyring 2024-09-14T03:38:23.146 INFO:tasks.ceph:Copying monmap to all nodes... 2024-09-14T03:38:23.146 DEBUG:teuthology.orchestra.run.smithi083:> set -ex 2024-09-14T03:38:23.146 DEBUG:teuthology.orchestra.run.smithi083:> dd if=/etc/ceph/ceph.keyring of=/dev/stdout 2024-09-14T03:38:23.162 DEBUG:teuthology.orchestra.run.smithi083:> set -ex 2024-09-14T03:38:23.162 DEBUG:teuthology.orchestra.run.smithi083:> dd if=/home/ubuntu/cephtest/ceph.monmap of=/dev/stdout 2024-09-14T03:38:23.217 INFO:tasks.ceph:Sending monmap to node ubuntu@smithi083.front.sepia.ceph.com 2024-09-14T03:38:23.217 DEBUG:teuthology.orchestra.run.smithi083:> set -ex 2024-09-14T03:38:23.217 DEBUG:teuthology.orchestra.run.smithi083:> sudo dd of=/etc/ceph/ceph.keyring 2024-09-14T03:38:23.217 DEBUG:teuthology.orchestra.run.smithi083:> sudo chmod 0644 /etc/ceph/ceph.keyring 2024-09-14T03:38:23.291 DEBUG:teuthology.orchestra.run.smithi083:> set -ex 2024-09-14T03:38:23.291 DEBUG:teuthology.orchestra.run.smithi083:> dd of=/home/ubuntu/cephtest/ceph.monmap 2024-09-14T03:38:23.346 INFO:tasks.ceph:Setting up mon nodes... 2024-09-14T03:38:23.347 INFO:tasks.ceph:Setting up mgr nodes... 2024-09-14T03:38:23.347 DEBUG:teuthology.orchestra.run.smithi083:> sudo mkdir -p /var/lib/ceph/mgr/ceph-x && sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=mgr.x /var/lib/ceph/mgr/ceph-x/keyring 2024-09-14T03:38:23.437 INFO:teuthology.orchestra.run.smithi083.stdout:creating /var/lib/ceph/mgr/ceph-x/keyring 2024-09-14T03:38:23.440 DEBUG:teuthology.orchestra.run.smithi083:> sudo mkdir -p /var/lib/ceph/mgr/ceph-y && sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=mgr.y /var/lib/ceph/mgr/ceph-y/keyring 2024-09-14T03:38:23.491 INFO:teuthology.orchestra.run.smithi083.stdout:creating /var/lib/ceph/mgr/ceph-y/keyring 2024-09-14T03:38:23.493 INFO:tasks.ceph:Setting up mds nodes... 2024-09-14T03:38:23.494 INFO:tasks.ceph_client:Setting up client nodes... 2024-09-14T03:38:23.494 INFO:tasks.ceph:Running mkfs on osd nodes... 2024-09-14T03:38:23.494 INFO:tasks.ceph:ctx.disk_config.remote_to_roles_to_dev: {Remote(name='ubuntu@smithi083.front.sepia.ceph.com'): {'osd.0': '/dev/vg_nvme/lv_1', 'osd.1': '/dev/vg_nvme/lv_2', 'osd.2': '/dev/vg_nvme/lv_3', 'osd.3': '/dev/vg_nvme/lv_4'}} 2024-09-14T03:38:23.494 DEBUG:teuthology.orchestra.run.smithi083:> sudo mkdir -p /var/lib/ceph/osd/ceph-0 2024-09-14T03:38:23.521 INFO:tasks.ceph:roles_to_devs: {'osd.0': '/dev/vg_nvme/lv_1', 'osd.1': '/dev/vg_nvme/lv_2', 'osd.2': '/dev/vg_nvme/lv_3', 'osd.3': '/dev/vg_nvme/lv_4'} 2024-09-14T03:38:23.521 INFO:tasks.ceph:role: osd.0 2024-09-14T03:38:23.521 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_1 on ubuntu@smithi083.front.sepia.ceph.com 2024-09-14T03:38:23.521 DEBUG:teuthology.orchestra.run.smithi083:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_1 2024-09-14T03:38:23.594 INFO:teuthology.orchestra.run.smithi083.stdout:meta-data=/dev/vg_nvme/lv_1 isize=2048 agcount=4, agsize=5859072 blks 2024-09-14T03:38:23.595 INFO:teuthology.orchestra.run.smithi083.stdout: = sectsz=512 attr=2, projid32bit=1 2024-09-14T03:38:23.595 INFO:teuthology.orchestra.run.smithi083.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2024-09-14T03:38:23.595 INFO:teuthology.orchestra.run.smithi083.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2024-09-14T03:38:23.595 INFO:teuthology.orchestra.run.smithi083.stdout:data = bsize=4096 blocks=23436288, imaxpct=25 2024-09-14T03:38:23.595 INFO:teuthology.orchestra.run.smithi083.stdout: = sunit=0 swidth=0 blks 2024-09-14T03:38:23.595 INFO:teuthology.orchestra.run.smithi083.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2024-09-14T03:38:23.595 INFO:teuthology.orchestra.run.smithi083.stdout:log =internal log bsize=4096 blocks=16384, version=2 2024-09-14T03:38:23.595 INFO:teuthology.orchestra.run.smithi083.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2024-09-14T03:38:23.595 INFO:teuthology.orchestra.run.smithi083.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2024-09-14T03:38:24.101 INFO:teuthology.orchestra.run.smithi083.stdout:Discarding blocks...Done. 2024-09-14T03:38:24.103 INFO:tasks.ceph:mount /dev/vg_nvme/lv_1 on ubuntu@smithi083.front.sepia.ceph.com -o noatime 2024-09-14T03:38:24.103 DEBUG:teuthology.orchestra.run.smithi083:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_1 /var/lib/ceph/osd/ceph-0 2024-09-14T03:38:24.208 DEBUG:teuthology.orchestra.run.smithi083:> sudo /sbin/restorecon /var/lib/ceph/osd/ceph-0 2024-09-14T03:38:24.239 DEBUG:teuthology.orchestra.run.smithi083:> sudo mkdir -p /var/lib/ceph/osd/ceph-1 2024-09-14T03:38:24.305 INFO:tasks.ceph:roles_to_devs: {'osd.0': '/dev/vg_nvme/lv_1', 'osd.1': '/dev/vg_nvme/lv_2', 'osd.2': '/dev/vg_nvme/lv_3', 'osd.3': '/dev/vg_nvme/lv_4'} 2024-09-14T03:38:24.305 INFO:tasks.ceph:role: osd.1 2024-09-14T03:38:24.306 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_2 on ubuntu@smithi083.front.sepia.ceph.com 2024-09-14T03:38:24.306 DEBUG:teuthology.orchestra.run.smithi083:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_2 2024-09-14T03:38:24.378 INFO:teuthology.orchestra.run.smithi083.stdout:meta-data=/dev/vg_nvme/lv_2 isize=2048 agcount=4, agsize=5859072 blks 2024-09-14T03:38:24.379 INFO:teuthology.orchestra.run.smithi083.stdout: = sectsz=512 attr=2, projid32bit=1 2024-09-14T03:38:24.379 INFO:teuthology.orchestra.run.smithi083.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2024-09-14T03:38:24.379 INFO:teuthology.orchestra.run.smithi083.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2024-09-14T03:38:24.379 INFO:teuthology.orchestra.run.smithi083.stdout:data = bsize=4096 blocks=23436288, imaxpct=25 2024-09-14T03:38:24.379 INFO:teuthology.orchestra.run.smithi083.stdout: = sunit=0 swidth=0 blks 2024-09-14T03:38:24.379 INFO:teuthology.orchestra.run.smithi083.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2024-09-14T03:38:24.379 INFO:teuthology.orchestra.run.smithi083.stdout:log =internal log bsize=4096 blocks=16384, version=2 2024-09-14T03:38:24.379 INFO:teuthology.orchestra.run.smithi083.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2024-09-14T03:38:24.379 INFO:teuthology.orchestra.run.smithi083.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2024-09-14T03:38:24.845 INFO:teuthology.orchestra.run.smithi083.stdout:Discarding blocks...Done. 2024-09-14T03:38:24.847 INFO:tasks.ceph:mount /dev/vg_nvme/lv_2 on ubuntu@smithi083.front.sepia.ceph.com -o noatime 2024-09-14T03:38:24.847 DEBUG:teuthology.orchestra.run.smithi083:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_2 /var/lib/ceph/osd/ceph-1 2024-09-14T03:38:24.942 DEBUG:teuthology.orchestra.run.smithi083:> sudo /sbin/restorecon /var/lib/ceph/osd/ceph-1 2024-09-14T03:38:24.971 DEBUG:teuthology.orchestra.run.smithi083:> sudo mkdir -p /var/lib/ceph/osd/ceph-2 2024-09-14T03:38:25.036 INFO:tasks.ceph:roles_to_devs: {'osd.0': '/dev/vg_nvme/lv_1', 'osd.1': '/dev/vg_nvme/lv_2', 'osd.2': '/dev/vg_nvme/lv_3', 'osd.3': '/dev/vg_nvme/lv_4'} 2024-09-14T03:38:25.037 INFO:tasks.ceph:role: osd.2 2024-09-14T03:38:25.037 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_3 on ubuntu@smithi083.front.sepia.ceph.com 2024-09-14T03:38:25.037 DEBUG:teuthology.orchestra.run.smithi083:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_3 2024-09-14T03:38:25.108 INFO:teuthology.orchestra.run.smithi083.stdout:meta-data=/dev/vg_nvme/lv_3 isize=2048 agcount=4, agsize=5859072 blks 2024-09-14T03:38:25.108 INFO:teuthology.orchestra.run.smithi083.stdout: = sectsz=512 attr=2, projid32bit=1 2024-09-14T03:38:25.108 INFO:teuthology.orchestra.run.smithi083.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2024-09-14T03:38:25.108 INFO:teuthology.orchestra.run.smithi083.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2024-09-14T03:38:25.108 INFO:teuthology.orchestra.run.smithi083.stdout:data = bsize=4096 blocks=23436288, imaxpct=25 2024-09-14T03:38:25.109 INFO:teuthology.orchestra.run.smithi083.stdout: = sunit=0 swidth=0 blks 2024-09-14T03:38:25.109 INFO:teuthology.orchestra.run.smithi083.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2024-09-14T03:38:25.109 INFO:teuthology.orchestra.run.smithi083.stdout:log =internal log bsize=4096 blocks=16384, version=2 2024-09-14T03:38:25.109 INFO:teuthology.orchestra.run.smithi083.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2024-09-14T03:38:25.109 INFO:teuthology.orchestra.run.smithi083.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2024-09-14T03:38:25.612 INFO:teuthology.orchestra.run.smithi083.stdout:Discarding blocks...Done. 2024-09-14T03:38:25.614 INFO:tasks.ceph:mount /dev/vg_nvme/lv_3 on ubuntu@smithi083.front.sepia.ceph.com -o noatime 2024-09-14T03:38:25.615 DEBUG:teuthology.orchestra.run.smithi083:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_3 /var/lib/ceph/osd/ceph-2 2024-09-14T03:38:25.730 DEBUG:teuthology.orchestra.run.smithi083:> sudo /sbin/restorecon /var/lib/ceph/osd/ceph-2 2024-09-14T03:38:25.759 DEBUG:teuthology.orchestra.run.smithi083:> sudo mkdir -p /var/lib/ceph/osd/ceph-3 2024-09-14T03:38:25.824 INFO:tasks.ceph:roles_to_devs: {'osd.0': '/dev/vg_nvme/lv_1', 'osd.1': '/dev/vg_nvme/lv_2', 'osd.2': '/dev/vg_nvme/lv_3', 'osd.3': '/dev/vg_nvme/lv_4'} 2024-09-14T03:38:25.824 INFO:tasks.ceph:role: osd.3 2024-09-14T03:38:25.824 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_4 on ubuntu@smithi083.front.sepia.ceph.com 2024-09-14T03:38:25.824 DEBUG:teuthology.orchestra.run.smithi083:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_4 2024-09-14T03:38:25.896 INFO:teuthology.orchestra.run.smithi083.stdout:meta-data=/dev/vg_nvme/lv_4 isize=2048 agcount=4, agsize=5859072 blks 2024-09-14T03:38:25.896 INFO:teuthology.orchestra.run.smithi083.stdout: = sectsz=512 attr=2, projid32bit=1 2024-09-14T03:38:25.896 INFO:teuthology.orchestra.run.smithi083.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2024-09-14T03:38:25.896 INFO:teuthology.orchestra.run.smithi083.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2024-09-14T03:38:25.896 INFO:teuthology.orchestra.run.smithi083.stdout:data = bsize=4096 blocks=23436288, imaxpct=25 2024-09-14T03:38:25.896 INFO:teuthology.orchestra.run.smithi083.stdout: = sunit=0 swidth=0 blks 2024-09-14T03:38:25.896 INFO:teuthology.orchestra.run.smithi083.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2024-09-14T03:38:25.896 INFO:teuthology.orchestra.run.smithi083.stdout:log =internal log bsize=4096 blocks=16384, version=2 2024-09-14T03:38:25.896 INFO:teuthology.orchestra.run.smithi083.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2024-09-14T03:38:25.897 INFO:teuthology.orchestra.run.smithi083.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2024-09-14T03:38:26.399 INFO:teuthology.orchestra.run.smithi083.stdout:Discarding blocks...Done. 2024-09-14T03:38:26.401 INFO:tasks.ceph:mount /dev/vg_nvme/lv_4 on ubuntu@smithi083.front.sepia.ceph.com -o noatime 2024-09-14T03:38:26.401 DEBUG:teuthology.orchestra.run.smithi083:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_4 /var/lib/ceph/osd/ceph-3 2024-09-14T03:38:26.528 DEBUG:teuthology.orchestra.run.smithi083:> sudo /sbin/restorecon /var/lib/ceph/osd/ceph-3 2024-09-14T03:38:26.559 DEBUG:teuthology.orchestra.run.smithi083:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster ceph --mkfs --mkkey -i 0 --monmap /home/ubuntu/cephtest/ceph.monmap 2024-09-14T03:38:26.656 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:26.654+0000 7fe733a0f640 -1 auth: error reading file: /var/lib/ceph/osd/ceph-0/keyring: can't open /var/lib/ceph/osd/ceph-0/keyring: (2) No such file or directory 2024-09-14T03:38:26.656 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:26.654+0000 7fe733a0f640 -1 created new key in keyring /var/lib/ceph/osd/ceph-0/keyring 2024-09-14T03:38:26.656 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:26.654+0000 7fe733a0f640 -1 bdev(0x55765c887800 /var/lib/ceph/osd/ceph-0/block) open stat got: (1) Operation not permitted 2024-09-14T03:38:26.656 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:26.655+0000 7fe733a0f640 -1 bluestore(/var/lib/ceph/osd/ceph-0) _read_fsid unparsable uuid 2024-09-14T03:38:30.562 DEBUG:teuthology.orchestra.run.smithi083:> sudo chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-14T03:38:30.592 DEBUG:teuthology.orchestra.run.smithi083:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster ceph --mkfs --mkkey -i 1 --monmap /home/ubuntu/cephtest/ceph.monmap 2024-09-14T03:38:30.682 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:30.680+0000 7fc9aee18640 -1 auth: error reading file: /var/lib/ceph/osd/ceph-1/keyring: can't open /var/lib/ceph/osd/ceph-1/keyring: (2) No such file or directory 2024-09-14T03:38:30.683 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:30.681+0000 7fc9aee18640 -1 created new key in keyring /var/lib/ceph/osd/ceph-1/keyring 2024-09-14T03:38:30.683 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:30.681+0000 7fc9aee18640 -1 bdev(0x55d2fb4d3800 /var/lib/ceph/osd/ceph-1/block) open stat got: (1) Operation not permitted 2024-09-14T03:38:30.683 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:30.681+0000 7fc9aee18640 -1 bluestore(/var/lib/ceph/osd/ceph-1) _read_fsid unparsable uuid 2024-09-14T03:38:34.595 DEBUG:teuthology.orchestra.run.smithi083:> sudo chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-14T03:38:34.624 DEBUG:teuthology.orchestra.run.smithi083:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster ceph --mkfs --mkkey -i 2 --monmap /home/ubuntu/cephtest/ceph.monmap 2024-09-14T03:38:34.715 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:34.713+0000 7f236100d640 -1 auth: error reading file: /var/lib/ceph/osd/ceph-2/keyring: can't open /var/lib/ceph/osd/ceph-2/keyring: (2) No such file or directory 2024-09-14T03:38:34.715 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:34.713+0000 7f236100d640 -1 created new key in keyring /var/lib/ceph/osd/ceph-2/keyring 2024-09-14T03:38:34.715 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:34.713+0000 7f236100d640 -1 bdev(0x56538e977800 /var/lib/ceph/osd/ceph-2/block) open stat got: (1) Operation not permitted 2024-09-14T03:38:34.715 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:34.713+0000 7f236100d640 -1 bluestore(/var/lib/ceph/osd/ceph-2) _read_fsid unparsable uuid 2024-09-14T03:38:38.377 DEBUG:teuthology.orchestra.run.smithi083:> sudo chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-14T03:38:38.406 DEBUG:teuthology.orchestra.run.smithi083:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster ceph --mkfs --mkkey -i 3 --monmap /home/ubuntu/cephtest/ceph.monmap 2024-09-14T03:38:38.496 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:38.495+0000 7f657ccd6640 -1 auth: error reading file: /var/lib/ceph/osd/ceph-3/keyring: can't open /var/lib/ceph/osd/ceph-3/keyring: (2) No such file or directory 2024-09-14T03:38:38.497 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:38.495+0000 7f657ccd6640 -1 created new key in keyring /var/lib/ceph/osd/ceph-3/keyring 2024-09-14T03:38:38.497 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:38.495+0000 7f657ccd6640 -1 bdev(0x55a6ae415800 /var/lib/ceph/osd/ceph-3/block) open stat got: (1) Operation not permitted 2024-09-14T03:38:38.497 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:38.495+0000 7f657ccd6640 -1 bluestore(/var/lib/ceph/osd/ceph-3) _read_fsid unparsable uuid 2024-09-14T03:38:42.156 DEBUG:teuthology.orchestra.run.smithi083:> sudo chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-14T03:38:42.185 INFO:tasks.ceph:Reading keys from all nodes... 2024-09-14T03:38:42.185 DEBUG:teuthology.orchestra.run.smithi083:> set -ex 2024-09-14T03:38:42.185 DEBUG:teuthology.orchestra.run.smithi083:> sudo dd if=/var/lib/ceph/mgr/ceph-x/keyring of=/dev/stdout 2024-09-14T03:38:42.253 DEBUG:teuthology.orchestra.run.smithi083:> set -ex 2024-09-14T03:38:42.253 DEBUG:teuthology.orchestra.run.smithi083:> sudo dd if=/var/lib/ceph/mgr/ceph-y/keyring of=/dev/stdout 2024-09-14T03:38:42.321 DEBUG:teuthology.orchestra.run.smithi083:> set -ex 2024-09-14T03:38:42.321 DEBUG:teuthology.orchestra.run.smithi083:> sudo dd if=/var/lib/ceph/osd/ceph-0/keyring of=/dev/stdout 2024-09-14T03:38:42.388 DEBUG:teuthology.orchestra.run.smithi083:> set -ex 2024-09-14T03:38:42.388 DEBUG:teuthology.orchestra.run.smithi083:> sudo dd if=/var/lib/ceph/osd/ceph-1/keyring of=/dev/stdout 2024-09-14T03:38:42.456 DEBUG:teuthology.orchestra.run.smithi083:> set -ex 2024-09-14T03:38:42.456 DEBUG:teuthology.orchestra.run.smithi083:> sudo dd if=/var/lib/ceph/osd/ceph-2/keyring of=/dev/stdout 2024-09-14T03:38:42.523 DEBUG:teuthology.orchestra.run.smithi083:> set -ex 2024-09-14T03:38:42.523 DEBUG:teuthology.orchestra.run.smithi083:> sudo dd if=/var/lib/ceph/osd/ceph-3/keyring of=/dev/stdout 2024-09-14T03:38:42.587 INFO:tasks.ceph:Adding keys to all mons... 2024-09-14T03:38:42.587 DEBUG:teuthology.orchestra.run.smithi083:> sudo tee -a /etc/ceph/ceph.keyring 2024-09-14T03:38:42.651 INFO:teuthology.orchestra.run.smithi083.stdout:[mgr.x] 2024-09-14T03:38:42.652 INFO:teuthology.orchestra.run.smithi083.stdout: key = AQAvBeVmfuoHGhAAOrrmasCPMuN9rbtkKokwiw== 2024-09-14T03:38:42.652 INFO:teuthology.orchestra.run.smithi083.stdout:[mgr.y] 2024-09-14T03:38:42.652 INFO:teuthology.orchestra.run.smithi083.stdout: key = AQAvBeVmZQU+HRAAE+PC71lpeey4XcVFfr3eiA== 2024-09-14T03:38:42.652 INFO:teuthology.orchestra.run.smithi083.stdout:[osd.0] 2024-09-14T03:38:42.652 INFO:teuthology.orchestra.run.smithi083.stdout: key = AQAyBeVmXksRJxAAUPSfSySfUmRMnhfC+yPjIg== 2024-09-14T03:38:42.652 INFO:teuthology.orchestra.run.smithi083.stdout:[osd.1] 2024-09-14T03:38:42.652 INFO:teuthology.orchestra.run.smithi083.stdout: key = AQA2BeVm0gajKBAAfFkPZqCvqfOUW1J4yehqlA== 2024-09-14T03:38:42.652 INFO:teuthology.orchestra.run.smithi083.stdout:[osd.2] 2024-09-14T03:38:42.652 INFO:teuthology.orchestra.run.smithi083.stdout: key = AQA6BeVmXteVKhAA0VbjyuKF7/ekRPseHHbNaQ== 2024-09-14T03:38:42.652 INFO:teuthology.orchestra.run.smithi083.stdout:[osd.3] 2024-09-14T03:38:42.652 INFO:teuthology.orchestra.run.smithi083.stdout: key = AQA+BeVmaQqRHRAAb8DWWMhZEtEhf+qpET9yQg== 2024-09-14T03:38:42.653 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=mgr.x --cap mon 'allow profile mgr' --cap osd 'allow *' --cap mds 'allow *' 2024-09-14T03:38:42.744 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=mgr.y --cap mon 'allow profile mgr' --cap osd 'allow *' --cap mds 'allow *' 2024-09-14T03:38:42.795 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.0 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2024-09-14T03:38:42.847 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.1 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2024-09-14T03:38:42.897 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.2 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2024-09-14T03:38:42.947 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.3 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2024-09-14T03:38:42.999 INFO:tasks.ceph:Running mkfs on mon nodes... 2024-09-14T03:38:43.000 DEBUG:teuthology.orchestra.run.smithi083:> sudo mkdir -p /var/lib/ceph/mon/ceph-a 2024-09-14T03:38:43.024 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-mon --cluster ceph --mkfs -i a --monmap /home/ubuntu/cephtest/ceph.monmap --keyring /etc/ceph/ceph.keyring 2024-09-14T03:38:43.120 DEBUG:teuthology.orchestra.run.smithi083:> sudo chown -R ceph:ceph /var/lib/ceph/mon/ceph-a 2024-09-14T03:38:43.144 DEBUG:teuthology.orchestra.run.smithi083:> sudo mkdir -p /var/lib/ceph/mon/ceph-b 2024-09-14T03:38:43.209 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-mon --cluster ceph --mkfs -i b --monmap /home/ubuntu/cephtest/ceph.monmap --keyring /etc/ceph/ceph.keyring 2024-09-14T03:38:43.306 DEBUG:teuthology.orchestra.run.smithi083:> sudo chown -R ceph:ceph /var/lib/ceph/mon/ceph-b 2024-09-14T03:38:43.330 DEBUG:teuthology.orchestra.run.smithi083:> sudo mkdir -p /var/lib/ceph/mon/ceph-c 2024-09-14T03:38:43.395 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-mon --cluster ceph --mkfs -i c --monmap /home/ubuntu/cephtest/ceph.monmap --keyring /etc/ceph/ceph.keyring 2024-09-14T03:38:43.490 DEBUG:teuthology.orchestra.run.smithi083:> sudo chown -R ceph:ceph /var/lib/ceph/mon/ceph-c 2024-09-14T03:38:43.515 DEBUG:teuthology.orchestra.run.smithi083:> rm -- /home/ubuntu/cephtest/ceph.monmap 2024-09-14T03:38:43.570 INFO:tasks.ceph:Starting mon daemons in cluster ceph... 2024-09-14T03:38:43.570 INFO:tasks.ceph.mon.a:Restarting daemon 2024-09-14T03:38:43.570 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mon -f --cluster ceph -i a 2024-09-14T03:38:43.612 INFO:tasks.ceph.mon.a:Started 2024-09-14T03:38:43.613 INFO:tasks.ceph.mon.b:Restarting daemon 2024-09-14T03:38:43.613 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mon -f --cluster ceph -i b 2024-09-14T03:38:43.616 INFO:tasks.ceph.mon.b:Started 2024-09-14T03:38:43.616 INFO:tasks.ceph.mon.c:Restarting daemon 2024-09-14T03:38:43.616 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mon -f --cluster ceph -i c 2024-09-14T03:38:43.618 INFO:tasks.ceph.mon.c:Started 2024-09-14T03:38:43.619 INFO:tasks.ceph:Starting mgr daemons in cluster ceph... 2024-09-14T03:38:43.619 INFO:tasks.ceph.mgr.x:Restarting daemon 2024-09-14T03:38:43.619 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mgr -f --cluster ceph -i x 2024-09-14T03:38:43.621 INFO:tasks.ceph.mgr.x:Started 2024-09-14T03:38:43.621 INFO:tasks.ceph.mgr.y:Restarting daemon 2024-09-14T03:38:43.621 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mgr -f --cluster ceph -i y 2024-09-14T03:38:43.623 INFO:tasks.ceph.mgr.y:Started 2024-09-14T03:38:43.623 INFO:tasks.ceph:setting config [client] client_mount_timeout = 600 2024-09-14T03:38:43.623 DEBUG:teuthology.orchestra.run.smithi083:> sudo ceph --cluster ceph config set client client_mount_timeout 600 2024-09-14T03:38:43.625 INFO:tasks.ceph:setting config [client] debug_client = 20 2024-09-14T03:38:43.626 DEBUG:teuthology.orchestra.run.smithi083:> sudo ceph --cluster ceph config set client debug_client 20 2024-09-14T03:38:43.669 INFO:tasks.ceph:setting config [client] debug_ms = 1 2024-09-14T03:38:43.669 DEBUG:teuthology.orchestra.run.smithi083:> sudo ceph --cluster ceph config set client debug_ms 1 2024-09-14T03:38:43.712 INFO:tasks.ceph:setting config [client] rados_mon_op_timeout = 900 2024-09-14T03:38:43.713 DEBUG:teuthology.orchestra.run.smithi083:> sudo ceph --cluster ceph config set client rados_mon_op_timeout 900 2024-09-14T03:38:43.757 INFO:tasks.ceph:setting config [client] rados_osd_op_timeout = 900 2024-09-14T03:38:43.757 DEBUG:teuthology.orchestra.run.smithi083:> sudo ceph --cluster ceph config set client rados_osd_op_timeout 900 2024-09-14T03:38:43.800 INFO:tasks.ceph:setting config [mds] debug_mds = 20 2024-09-14T03:38:43.800 DEBUG:teuthology.orchestra.run.smithi083:> sudo ceph --cluster ceph config set mds debug_mds 20 2024-09-14T03:38:43.843 INFO:tasks.ceph:setting config [mds] debug_mds_balancer = 20 2024-09-14T03:38:43.843 DEBUG:teuthology.orchestra.run.smithi083:> sudo ceph --cluster ceph config set mds debug_mds_balancer 20 2024-09-14T03:38:43.887 INFO:tasks.ceph:setting config [mds] debug_ms = 2 2024-09-14T03:38:43.887 DEBUG:teuthology.orchestra.run.smithi083:> sudo ceph --cluster ceph config set mds debug_ms 2 2024-09-14T03:38:43.931 INFO:tasks.ceph:setting config [mds] mds_debug_frag = True 2024-09-14T03:38:43.931 DEBUG:teuthology.orchestra.run.smithi083:> sudo ceph --cluster ceph config set mds mds_debug_frag True 2024-09-14T03:38:43.975 INFO:tasks.ceph:setting config [mds] mds_debug_scatterstat = True 2024-09-14T03:38:43.975 DEBUG:teuthology.orchestra.run.smithi083:> sudo ceph --cluster ceph config set mds mds_debug_scatterstat True 2024-09-14T03:38:44.018 INFO:tasks.ceph:setting config [mds] mds_op_complaint_time = 180 2024-09-14T03:38:44.018 DEBUG:teuthology.orchestra.run.smithi083:> sudo ceph --cluster ceph config set mds mds_op_complaint_time 180 2024-09-14T03:38:44.063 INFO:tasks.ceph:setting config [mds] mds_verify_scatter = True 2024-09-14T03:38:44.063 DEBUG:teuthology.orchestra.run.smithi083:> sudo ceph --cluster ceph config set mds mds_verify_scatter True 2024-09-14T03:38:44.106 INFO:tasks.ceph:setting config [mds] osd_op_complaint_time = 180 2024-09-14T03:38:44.106 DEBUG:teuthology.orchestra.run.smithi083:> sudo ceph --cluster ceph config set mds osd_op_complaint_time 180 2024-09-14T03:38:44.153 INFO:tasks.ceph:setting config [mds] rados_mon_op_timeout = 900 2024-09-14T03:38:44.153 DEBUG:teuthology.orchestra.run.smithi083:> sudo ceph --cluster ceph config set mds rados_mon_op_timeout 900 2024-09-14T03:38:44.181 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.180+0000 7f1651bc9640 1 -- 172.21.15.83:0/1091178770 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_command([{prefix=config set, name=debug_mds}] v 0) -- 0x7f164c1c57e0 con 0x7f164c12dbc0 2024-09-14T03:38:44.184 INFO:tasks.ceph:setting config [mds] rados_osd_op_timeout = 900 2024-09-14T03:38:44.185 DEBUG:teuthology.orchestra.run.smithi083:> sudo ceph --cluster ceph config set mds rados_osd_op_timeout 900 2024-09-14T03:38:44.185 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.182+0000 7fcfdffff640 1 -- 172.21.15.83:0/3178314671 <== mon.2 v2:172.21.15.83:3302/0 8 ==== mon_command_ack([{prefix=config set, name=debug_ms}]=0 v3)=0 v3) ==== 109+0+0 (secure 0 0 0) 0x7fcff0010b70 con 0x7fcff8123690 2024-09-14T03:38:44.186 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.184+0000 7fd8137fe640 1 -- 172.21.15.83:0/1151879100 <== mon.0 v2:172.21.15.83:3300/0 8 ==== mon_command_ack([{prefix=config set, name=debug_client}]=0 v4)=0 v4) ==== 114+0+0 (secure 0 0 0) 0x7fd82401a9b0 con 0x7fd82c0735c0 2024-09-14T03:38:44.186 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.185+0000 7fcfddffb640 1 -- 172.21.15.83:0/3178314671 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fcff8123690 msgr2=0x7fcff81c92a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:38:44.187 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.185+0000 7fcfddffb640 1 --2- 172.21.15.83:0/3178314671 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fcff8123690 0x7fcff81c92a0 secure :-1 s=READY pgs=7 cs=0 l=1 rev1=1 crypto rx=0x7fcff809abe0 tx=0x7fcff000cb00 comp rx=0 tx=0).stop 2024-09-14T03:38:44.189 INFO:tasks.ceph:setting config [mon] mon_op_complaint_time = 120 2024-09-14T03:38:44.189 DEBUG:teuthology.orchestra.run.smithi083:> sudo ceph --cluster ceph config set mon mon_op_complaint_time 120 2024-09-14T03:38:44.189 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.185+0000 7fd8137fe640 1 -- 172.21.15.83:0/1151879100 <== mon.0 v2:172.21.15.83:3300/0 9 ==== config(2 keys) ==== 48+0+0 (secure 0 0 0) 0x7fd82401ac10 con 0x7fd82c0735c0 2024-09-14T03:38:44.190 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.186+0000 7fcfddffb640 1 -- 172.21.15.83:0/3178314671 shutdown_connections 2024-09-14T03:38:44.190 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.186+0000 7fcfddffb640 1 --2- 172.21.15.83:0/3178314671 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fcff8123690 0x7fcff81c92a0 unknown :-1 s=CLOSED pgs=7 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:44.190 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.186+0000 7fcfddffb640 1 --2- 172.21.15.83:0/3178314671 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fcff81256f0 0x7fcff8123150 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:44.190 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.186+0000 7fcfddffb640 1 --2- 172.21.15.83:0/3178314671 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fcfe0004800 0x7fcff8122c10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:44.190 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.186+0000 7fcfddffb640 1 -- 172.21.15.83:0/3178314671 >> 172.21.15.83:0/3178314671 conn(0x7fcff809d2b0 msgr2=0x7fcff808d6a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:38:44.190 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.186+0000 7f1633fff640 1 -- 172.21.15.83:0/1091178770 <== mon.0 v2:172.21.15.83:3300/0 8 ==== config(2 keys) ==== 48+0+0 (secure 0 0 0) 0x7f1640003870 con 0x7f164c12dbc0 2024-09-14T03:38:44.191 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.187+0000 7fcfddffb640 1 -- 172.21.15.83:0/3178314671 shutdown_connections 2024-09-14T03:38:44.191 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.187+0000 7fcfddffb640 1 -- 172.21.15.83:0/3178314671 wait complete. 2024-09-14T03:38:44.191 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.189+0000 7fbc09ffb640 1 -- 172.21.15.83:0/402725944 <== mon.2 v2:172.21.15.83:3302/0 8 ==== config(2 keys) ==== 48+0+0 (secure 0 0 0) 0x7fbc0c0042a0 con 0x7fbc1412c400 2024-09-14T03:38:44.192 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.188+0000 7f4263d8a640 1 -- 172.21.15.83:0/4015085724 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_command([{prefix=config set, name=rados_osd_op_timeout}] v 0) -- 0x7f425c074000 con 0x7f4244004800 2024-09-14T03:38:44.195 INFO:tasks.ceph:setting config [mon] mon_warn_on_pool_no_app = False 2024-09-14T03:38:44.196 DEBUG:teuthology.orchestra.run.smithi083:> sudo ceph --cluster ceph config set mon mon_warn_on_pool_no_app False 2024-09-14T03:38:44.196 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.193+0000 7fd831733640 1 -- 172.21.15.83:0/1151879100 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fd82c0735c0 msgr2=0x7fd82c12dd50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:38:44.196 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.193+0000 7fd831733640 1 --2- 172.21.15.83:0/1151879100 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fd82c0735c0 0x7fd82c12dd50 secure :-1 s=READY pgs=13 cs=0 l=1 rev1=1 crypto rx=0x7fd82c09aec0 tx=0x7fd82400eac0 comp rx=0 tx=0).stop 2024-09-14T03:38:44.196 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.194+0000 7fd831733640 1 -- 172.21.15.83:0/1151879100 shutdown_connections 2024-09-14T03:38:44.196 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.194+0000 7fd831733640 1 --2- 172.21.15.83:0/1151879100 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fd82c0735c0 0x7fd82c12dd50 unknown :-1 s=CLOSED pgs=13 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:44.197 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.194+0000 7fd831733640 1 --2- 172.21.15.83:0/1151879100 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fd82c06ee10 0x7fd82c12d810 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:44.197 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.194+0000 7fd831733640 1 --2- 172.21.15.83:0/1151879100 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fd8140039d0 0x7fd82c12d2d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:44.197 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.194+0000 7fd831733640 1 -- 172.21.15.83:0/1151879100 >> 172.21.15.83:0/1151879100 conn(0x7fd82c09d620 msgr2=0x7fd82c094d10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:38:44.197 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.194+0000 7fd831733640 1 -- 172.21.15.83:0/1151879100 shutdown_connections 2024-09-14T03:38:44.197 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.194+0000 7fd831733640 1 -- 172.21.15.83:0/1151879100 wait complete. 2024-09-14T03:38:44.200 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.198+0000 7f1633fff640 1 -- 172.21.15.83:0/1091178770 <== mon.0 v2:172.21.15.83:3300/0 9 ==== mon_command_ack([{prefix=config set, name=debug_mds}]=0 v5)=0 v5) ==== 108+0+0 (secure 0 0 0) 0x7f1640003b30 con 0x7f164c12dbc0 2024-09-14T03:38:44.203 INFO:tasks.ceph:setting config [osd] osd_op_complaint_time = 180 2024-09-14T03:38:44.203 DEBUG:teuthology.orchestra.run.smithi083:> sudo ceph --cluster ceph config set osd osd_op_complaint_time 180 2024-09-14T03:38:44.205 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.203+0000 7f1651bc9640 1 -- 172.21.15.83:0/1091178770 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f164c12dbc0 msgr2=0x7f164c12dfe0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:38:44.205 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.203+0000 7f1651bc9640 1 --2- 172.21.15.83:0/1091178770 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f164c12dbc0 0x7f164c12dfe0 secure :-1 s=READY pgs=12 cs=0 l=1 rev1=1 crypto rx=0x7f164000bc00 tx=0x7f1640004730 comp rx=0 tx=0).stop 2024-09-14T03:38:44.205 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.203+0000 7f1651bc9640 1 -- 172.21.15.83:0/1091178770 shutdown_connections 2024-09-14T03:38:44.205 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.203+0000 7f1651bc9640 1 --2- 172.21.15.83:0/1091178770 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f164c1c2eb0 0x7f164c1c52a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:44.205 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.203+0000 7f1651bc9640 1 --2- 172.21.15.83:0/1091178770 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f164c12dbc0 0x7f164c12dfe0 unknown :-1 s=CLOSED pgs=12 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:44.206 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.203+0000 7f1651bc9640 1 --2- 172.21.15.83:0/1091178770 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f1634004800 0x7f164c12d680 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:44.206 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.203+0000 7f1651bc9640 1 -- 172.21.15.83:0/1091178770 >> 172.21.15.83:0/1091178770 conn(0x7f164c09d2b0 msgr2=0x7f164c129ca0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:38:44.206 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.204+0000 7f1651bc9640 1 -- 172.21.15.83:0/1091178770 shutdown_connections 2024-09-14T03:38:44.206 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.204+0000 7f1651bc9640 1 -- 172.21.15.83:0/1091178770 wait complete. 2024-09-14T03:38:44.207 DEBUG:tasks.ceph:set 18 configs 2024-09-14T03:38:44.208 DEBUG:tasks.ceph:waiting for RemoteProcess(client=, args=['sudo', 'ceph', '--cluster', 'ceph', 'config', 'set', 'client', 'client_mount_timeout', '600'], check_status=True, hostname='smithi083') 2024-09-14T03:38:44.209 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.206+0000 7f99c7fff640 1 -- 172.21.15.83:0/1227517626 <== mon.1 v2:172.21.15.83:3301/0 8 ==== config(2 keys) ==== 48+0+0 (secure 0 0 0) 0x7f99d00108b0 con 0x7f99e0093e70 2024-09-14T03:38:44.211 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.194+0000 7f425a7fc640 1 -- 172.21.15.83:0/4015085724 <== mon.1 v2:172.21.15.83:3301/0 8 ==== config(2 keys) ==== 48+0+0 (secure 0 0 0) 0x7f424c010b20 con 0x7f4244004800 2024-09-14T03:38:44.215 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.214+0000 7f9a9a7fc640 1 -- 172.21.15.83:0/2429276685 <== mon.0 v2:172.21.15.83:3300/0 9 ==== config(3 keys) ==== 79+0+0 (secure 0 0 0) 0x7f9aa0031be0 con 0x7f9aa412a220 2024-09-14T03:38:44.220 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.219+0000 7f99c7fff640 1 -- 172.21.15.83:0/1227517626 <== mon.1 v2:172.21.15.83:3301/0 9 ==== config(3 keys) ==== 79+0+0 (secure 0 0 0) 0x7f99d0010a60 con 0x7f99e0093e70 2024-09-14T03:38:44.220 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.218+0000 7f425a7fc640 1 -- 172.21.15.83:0/4015085724 <== mon.1 v2:172.21.15.83:3301/0 9 ==== config(3 keys) ==== 79+0+0 (secure 0 0 0) 0x7f424c024430 con 0x7f4244004800 2024-09-14T03:38:44.224 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.222+0000 7fbc09ffb640 1 -- 172.21.15.83:0/402725944 <== mon.2 v2:172.21.15.83:3302/0 9 ==== config(3 keys) ==== 79+0+0 (secure 0 0 0) 0x7fbc0c00ead0 con 0x7fbc1412c400 2024-09-14T03:38:44.226 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.224+0000 7f425a7fc640 1 -- 172.21.15.83:0/4015085724 <== mon.1 v2:172.21.15.83:3301/0 10 ==== mon_command_ack([{prefix=config set, name=rados_osd_op_timeout}]=0 v6)=0 v6) ==== 123+0+0 (secure 0 0 0) 0x7f425c126080 con 0x7f4244004800 2024-09-14T03:38:44.228 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.227+0000 7f4263d8a640 1 -- 172.21.15.83:0/4015085724 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f4244004800 msgr2=0x7f425c122ff0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:38:44.228 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.227+0000 7f4263d8a640 1 --2- 172.21.15.83:0/4015085724 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f4244004800 0x7f425c122ff0 secure :-1 s=READY pgs=7 cs=0 l=1 rev1=1 crypto rx=0x7f424c00d980 tx=0x7f424c00de40 comp rx=0 tx=0).stop 2024-09-14T03:38:44.230 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.229+0000 7f4263d8a640 1 -- 172.21.15.83:0/4015085724 shutdown_connections 2024-09-14T03:38:44.230 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.229+0000 7f4263d8a640 1 --2- 172.21.15.83:0/4015085724 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f425c1c0950 0x7f425c1c0db0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:44.230 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.229+0000 7f4263d8a640 1 --2- 172.21.15.83:0/4015085724 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f425c123530 0x7f425c123970 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:44.230 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.229+0000 7f4263d8a640 1 --2- 172.21.15.83:0/4015085724 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f4244004800 0x7f425c122ff0 unknown :-1 s=CLOSED pgs=7 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:44.230 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.229+0000 7f4263d8a640 1 -- 172.21.15.83:0/4015085724 >> 172.21.15.83:0/4015085724 conn(0x7f425c099770 msgr2=0x7f425c09a7a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:38:44.231 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.229+0000 7f4263d8a640 1 -- 172.21.15.83:0/4015085724 shutdown_connections 2024-09-14T03:38:44.231 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.229+0000 7f4263d8a640 1 -- 172.21.15.83:0/4015085724 wait complete. 2024-09-14T03:38:44.237 INFO:tasks.ceph.mgr.x.smithi083.stderr:2024-09-14T03:38:44.235+0000 7f5470a36040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-14T03:38:44.256 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.254+0000 7fbbf77fe640 1 -- 172.21.15.83:0/402725944 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_command([{prefix=config set, name=rados_mon_op_timeout}] v 0) -- 0x7fbbf0005100 con 0x7fbc1412c400 2024-09-14T03:38:44.269 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.266+0000 7f9a9a7fc640 1 -- 172.21.15.83:0/2429276685 <== mon.0 v2:172.21.15.83:3300/0 10 ==== config(4 keys) ==== 110+0+0 (secure 0 0 0) 0x7f9aa004e430 con 0x7f9aa412a220 2024-09-14T03:38:44.274 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.273+0000 7fbc09ffb640 1 -- 172.21.15.83:0/402725944 <== mon.2 v2:172.21.15.83:3302/0 10 ==== config(4 keys) ==== 110+0+0 (secure 0 0 0) 0x7fbc0c00edb0 con 0x7fbc1412c400 2024-09-14T03:38:44.274 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.273+0000 7fbc09ffb640 1 -- 172.21.15.83:0/402725944 <== mon.2 v2:172.21.15.83:3302/0 11 ==== mon_command_ack([{prefix=config set, name=rados_mon_op_timeout}]=0 v7)=0 v7) ==== 123+0+0 (secure 0 0 0) 0x7fbc0c023540 con 0x7fbc1412c400 2024-09-14T03:38:44.280 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.279+0000 7f99c7fff640 1 -- 172.21.15.83:0/1227517626 <== mon.1 v2:172.21.15.83:3301/0 10 ==== config(4 keys) ==== 110+0+0 (secure 0 0 0) 0x7f99d000f3e0 con 0x7f99e0093e70 2024-09-14T03:38:44.281 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.279+0000 7fbbf77fe640 1 -- 172.21.15.83:0/402725944 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fbc1412c400 msgr2=0x7fbc14127670 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:38:44.281 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.279+0000 7fbbf77fe640 1 --2- 172.21.15.83:0/402725944 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fbc1412c400 0x7fbc14127670 secure :-1 s=READY pgs=9 cs=0 l=1 rev1=1 crypto rx=0x7fbc0c00fa00 tx=0x7fbc0c00fec0 comp rx=0 tx=0).stop 2024-09-14T03:38:44.281 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.279+0000 7fbbf77fe640 1 -- 172.21.15.83:0/402725944 shutdown_connections 2024-09-14T03:38:44.281 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.279+0000 7fbbf77fe640 1 --2- 172.21.15.83:0/402725944 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fbc1412c400 0x7fbc14127670 unknown :-1 s=CLOSED pgs=9 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:44.281 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.279+0000 7fbbf77fe640 1 --2- 172.21.15.83:0/402725944 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fbc1412b9b0 0x7fbc14127130 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:44.281 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.279+0000 7fbbf77fe640 1 --2- 172.21.15.83:0/402725944 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fbc141289b0 0x7fbc14126bf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:44.281 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.279+0000 7fbbf77fe640 1 -- 172.21.15.83:0/402725944 >> 172.21.15.83:0/402725944 conn(0x7fbc1409e280 msgr2=0x7fbc1412b260 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:38:44.282 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.280+0000 7fbbf77fe640 1 -- 172.21.15.83:0/402725944 shutdown_connections 2024-09-14T03:38:44.282 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.280+0000 7fbbf77fe640 1 -- 172.21.15.83:0/402725944 wait complete. 2024-09-14T03:38:44.304 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.303+0000 7fd4ef47e640 1 -- 172.21.15.83:0/908120584 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fd4e8137990 msgr2=0x7fd4e8139d80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:38:44.305 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.303+0000 7fd4ef47e640 1 --2- 172.21.15.83:0/908120584 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fd4e8137990 0x7fd4e8139d80 secure :-1 s=READY pgs=10 cs=0 l=1 rev1=1 crypto rx=0x7fd4e0009be0 tx=0x7fd4e002a680 comp rx=0 tx=0).stop 2024-09-14T03:38:44.305 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.303+0000 7fd4ef47e640 1 -- 172.21.15.83:0/908120584 shutdown_connections 2024-09-14T03:38:44.305 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.303+0000 7fd4ef47e640 1 --2- 172.21.15.83:0/908120584 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fd4e8137990 0x7fd4e8139d80 unknown :-1 s=CLOSED pgs=10 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:44.306 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.303+0000 7fd4ef47e640 1 --2- 172.21.15.83:0/908120584 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fd4e8093470 0x7fd4e8093850 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:44.306 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.303+0000 7fd4ef47e640 1 -- 172.21.15.83:0/908120584 >> 172.21.15.83:0/908120584 conn(0x7fd4e809c1c0 msgr2=0x7fd4e809c5d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:38:44.309 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.308+0000 7fd4ef47e640 1 -- 172.21.15.83:0/908120584 shutdown_connections 2024-09-14T03:38:44.315 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.313+0000 7fd4ef47e640 1 -- 172.21.15.83:0/908120584 wait complete. 2024-09-14T03:38:44.316 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.314+0000 7fd4ef47e640 1 Processor -- start 2024-09-14T03:38:44.318 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.316+0000 7f9aabfdb640 1 -- 172.21.15.83:0/2429276685 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_command([{prefix=config set, name=debug_ms}] v 0) -- 0x7f9a800052f0 con 0x7f9aa412a220 2024-09-14T03:38:44.327 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.326+0000 7f9a9a7fc640 1 -- 172.21.15.83:0/2429276685 <== mon.0 v2:172.21.15.83:3300/0 11 ==== mon_command_ack([{prefix=config set, name=debug_ms}]=0 v8)=0 v8) ==== 106+0+0 (secure 0 0 0) 0x7f9aa004e7d0 con 0x7f9aa412a220 2024-09-14T03:38:44.330 INFO:tasks.ceph.mgr.x.smithi083.stderr:2024-09-14T03:38:44.329+0000 7f5470a36040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-14T03:38:44.332 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.331+0000 7f9a87fff640 1 -- 172.21.15.83:0/2429276685 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f9aa412a220 msgr2=0x7f9aa4123070 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:38:44.332 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.331+0000 7f9a87fff640 1 --2- 172.21.15.83:0/2429276685 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f9aa412a220 0x7f9aa4123070 secure :-1 s=READY pgs=16 cs=0 l=1 rev1=1 crypto rx=0x7f9aa0009a50 tx=0x7f9aa00046c0 comp rx=0 tx=0).stop 2024-09-14T03:38:44.333 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.331+0000 7f9a87fff640 1 -- 172.21.15.83:0/2429276685 shutdown_connections 2024-09-14T03:38:44.333 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.331+0000 7f9a87fff640 1 --2- 172.21.15.83:0/2429276685 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f9aa412ab40 0x7f9aa41235b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:44.333 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.331+0000 7f9a87fff640 1 --2- 172.21.15.83:0/2429276685 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f9aa412a220 0x7f9aa4123070 unknown :-1 s=CLOSED pgs=16 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:44.333 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.331+0000 7f9a87fff640 1 --2- 172.21.15.83:0/2429276685 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f9aa40965f0 0x7f9aa4122b30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:44.333 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.331+0000 7f9a87fff640 1 -- 172.21.15.83:0/2429276685 >> 172.21.15.83:0/2429276685 conn(0x7f9aa409d070 msgr2=0x7f9aa4097440 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:38:44.333 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.315+0000 7fd4ef47e640 1 -- start start 2024-09-14T03:38:44.333 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.316+0000 7fd4ef47e640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fd4e8093470 0x7fd4e808c400 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:38:44.333 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.316+0000 7fd4ef47e640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fd4e8093d90 0x7fd4e808c940 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:38:44.334 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.316+0000 7fd4ef47e640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fd4e8137990 0x7fd4e808ce80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:38:44.334 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.316+0000 7fd4ef47e640 1 -- --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7fd4e813ab30 con 0x7fd4e8093d90 2024-09-14T03:38:44.334 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.316+0000 7fd4ef47e640 1 -- --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7fd4e813a9b0 con 0x7fd4e8093470 2024-09-14T03:38:44.334 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.316+0000 7fd4ef47e640 1 -- --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7fd4e813acb0 con 0x7fd4e8137990 2024-09-14T03:38:44.334 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.316+0000 7fd4edc7b640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fd4e8093d90 0x7fd4e808c940 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:38:44.334 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.316+0000 7fd4edc7b640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fd4e8093d90 0x7fd4e808c940 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3300/0 says I am v2:172.21.15.83:43826/0 (socket says 172.21.15.83:43826) 2024-09-14T03:38:44.334 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.316+0000 7fd4edc7b640 1 -- 172.21.15.83:0/1427676856 learned_addr learned my addr 172.21.15.83:0/1427676856 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:38:44.334 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.330+0000 7fd4edc7b640 1 -- 172.21.15.83:0/1427676856 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fd4e8137990 msgr2=0x7fd4e808ce80 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-09-14T03:38:44.334 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.330+0000 7fd4edc7b640 1 --2- 172.21.15.83:0/1427676856 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fd4e8137990 0x7fd4e808ce80 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:44.334 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.330+0000 7fd4edc7b640 1 -- 172.21.15.83:0/1427676856 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fd4e8093470 msgr2=0x7fd4e808c400 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-09-14T03:38:44.334 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.330+0000 7fd4edc7b640 1 --2- 172.21.15.83:0/1427676856 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fd4e8093470 0x7fd4e808c400 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:44.334 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.330+0000 7fd4edc7b640 1 -- 172.21.15.83:0/1427676856 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fd4e808e770 con 0x7fd4e8093d90 2024-09-14T03:38:44.334 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.331+0000 7fd4edc7b640 1 --2- 172.21.15.83:0/1427676856 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fd4e8093d90 0x7fd4e808c940 secure :-1 s=READY pgs=19 cs=0 l=1 rev1=1 crypto rx=0x7fd4e400c870 tx=0x7fd4e400cd30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:38:44.335 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.331+0000 7f9a87fff640 1 -- 172.21.15.83:0/2429276685 shutdown_connections 2024-09-14T03:38:44.335 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.331+0000 7f9a87fff640 1 -- 172.21.15.83:0/2429276685 wait complete. 2024-09-14T03:38:44.335 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.332+0000 7fd4deffd640 1 -- 172.21.15.83:0/1427676856 <== mon.0 v2:172.21.15.83:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd4e4017070 con 0x7fd4e8093d90 2024-09-14T03:38:44.340 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.332+0000 7fd4ef47e640 1 -- 172.21.15.83:0/1427676856 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fd4e808de90 con 0x7fd4e8093d90 2024-09-14T03:38:44.340 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.332+0000 7fd4ef47e640 1 -- 172.21.15.83:0/1427676856 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fd4e812dc50 con 0x7fd4e8093d90 2024-09-14T03:38:44.340 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.334+0000 7fd4deffd640 1 -- 172.21.15.83:0/1427676856 <== mon.0 v2:172.21.15.83:3300/0 2 ==== config(4 keys) ==== 110+0+0 (secure 0 0 0) 0x7fd4e4005340 con 0x7fd4e8093d90 2024-09-14T03:38:44.341 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.334+0000 7fd4deffd640 1 -- 172.21.15.83:0/1427676856 <== mon.0 v2:172.21.15.83:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd4e4003830 con 0x7fd4e8093d90 2024-09-14T03:38:44.345 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.343+0000 7fd4ef47e640 1 -- 172.21.15.83:0/1427676856 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fd4d0005290 con 0x7fd4e8093d90 2024-09-14T03:38:44.345 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.344+0000 7fd4deffd640 1 -- 172.21.15.83:0/1427676856 <== mon.0 v2:172.21.15.83:3300/0 4 ==== mgrmap(e 1) ==== 807+0+0 (secure 0 0 0) 0x7fd4e401a070 con 0x7fd4e8093d90 2024-09-14T03:38:44.345 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.344+0000 7fd4deffd640 1 -- 172.21.15.83:0/1427676856 <== mon.0 v2:172.21.15.83:3300/0 5 ==== osd_map(1..1 src has 1..1) ==== 725+0+0 (secure 0 0 0) 0x7fd4e401c440 con 0x7fd4e8093d90 2024-09-14T03:38:44.357 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.353+0000 7fd4deffd640 1 -- 172.21.15.83:0/1427676856 <== mon.0 v2:172.21.15.83:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+74432 (secure 0 0 0) 0x7fd4e400a5e0 con 0x7fd4e8093d90 2024-09-14T03:38:44.369 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.367+0000 7f9cd4765640 1 -- 172.21.15.83:0/2964623190 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f9ccc0965f0 msgr2=0x7f9ccc096a60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:38:44.369 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.367+0000 7f9cd4765640 1 --2- 172.21.15.83:0/2964623190 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f9ccc0965f0 0x7f9ccc096a60 secure :-1 s=READY pgs=12 cs=0 l=1 rev1=1 crypto rx=0x7f9cc4005fd0 tx=0x7f9cc402b420 comp rx=0 tx=0).stop 2024-09-14T03:38:44.369 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.368+0000 7f9cd4765640 1 -- 172.21.15.83:0/2964623190 shutdown_connections 2024-09-14T03:38:44.369 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.368+0000 7f9cd4765640 1 --2- 172.21.15.83:0/2964623190 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f9ccc06fb70 0x7f9ccc0960b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:44.369 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.368+0000 7f9cd4765640 1 --2- 172.21.15.83:0/2964623190 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f9ccc0965f0 0x7f9ccc096a60 unknown :-1 s=CLOSED pgs=12 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:44.369 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.368+0000 7f9cd4765640 1 --2- 172.21.15.83:0/2964623190 >> v2:172.21.15.83:3300/0 conn(0x7f9ccc06f250 0x7f9ccc06f630 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:44.369 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.368+0000 7f9cd4765640 1 -- 172.21.15.83:0/2964623190 >> 172.21.15.83:0/2964623190 conn(0x7f9ccc09d070 msgr2=0x7f9ccc099540 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:38:44.373 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.368+0000 7f9cd4765640 1 -- 172.21.15.83:0/2964623190 shutdown_connections 2024-09-14T03:38:44.373 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.368+0000 7f9cd4765640 1 -- 172.21.15.83:0/2964623190 wait complete. 2024-09-14T03:38:44.373 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.369+0000 7f9cd4765640 1 Processor -- start 2024-09-14T03:38:44.373 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.370+0000 7f9cd4765640 1 -- start start 2024-09-14T03:38:44.373 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.370+0000 7f9cd4765640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f9ccc06f250 0x7f9ccc127040 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:38:44.373 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.370+0000 7f9cd4765640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f9ccc06fb70 0x7f9ccc127580 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:38:44.373 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.371+0000 7f9cd1d00640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f9ccc06fb70 0x7f9ccc127580 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:38:44.373 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.371+0000 7f9cd1d00640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f9ccc06fb70 0x7f9ccc127580 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3302/0 says I am v2:172.21.15.83:60264/0 (socket says 172.21.15.83:60264) 2024-09-14T03:38:44.373 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.371+0000 7f9cd2501640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f9ccc06f250 0x7f9ccc127040 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:38:44.373 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.371+0000 7f9cd2501640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f9ccc06f250 0x7f9ccc127040 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3300/0 says I am v2:172.21.15.83:43864/0 (socket says 172.21.15.83:43864) 2024-09-14T03:38:44.373 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.371+0000 7f9cd1d00640 1 -- 172.21.15.83:0/810846788 learned_addr learned my addr 172.21.15.83:0/810846788 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:38:44.373 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.371+0000 7f9cd4765640 1 --2- 172.21.15.83:0/810846788 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f9ccc0965f0 0x7f9ccc127ac0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:38:44.375 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.372+0000 7f9cd2d02640 1 --2- 172.21.15.83:0/810846788 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f9ccc0965f0 0x7f9ccc127ac0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:38:44.375 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.372+0000 7f9cd4765640 1 -- 172.21.15.83:0/810846788 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7f9ccc1335a0 con 0x7f9ccc06f250 2024-09-14T03:38:44.375 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.372+0000 7f9cd4765640 1 -- 172.21.15.83:0/810846788 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7f9ccc133420 con 0x7f9ccc0965f0 2024-09-14T03:38:44.375 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.372+0000 7f9cd4765640 1 -- 172.21.15.83:0/810846788 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7f9ccc133720 con 0x7f9ccc06fb70 2024-09-14T03:38:44.375 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.373+0000 7f9cd1d00640 1 -- 172.21.15.83:0/810846788 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f9ccc0965f0 msgr2=0x7f9ccc127ac0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:38:44.375 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.373+0000 7f9cd1d00640 1 --2- 172.21.15.83:0/810846788 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f9ccc0965f0 0x7f9ccc127ac0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:44.375 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.373+0000 7f9cd1d00640 1 -- 172.21.15.83:0/810846788 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f9ccc06f250 msgr2=0x7f9ccc127040 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:38:44.375 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.373+0000 7f9cd1d00640 1 --2- 172.21.15.83:0/810846788 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f9ccc06f250 0x7f9ccc127040 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:44.375 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.373+0000 7f9cd1d00640 1 -- 172.21.15.83:0/810846788 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f9ccc129b00 con 0x7f9ccc06fb70 2024-09-14T03:38:44.375 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.373+0000 7f9cd2d02640 1 --2- 172.21.15.83:0/810846788 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f9ccc0965f0 0x7f9ccc127ac0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:38:44.375 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.373+0000 7f9cd1d00640 1 --2- 172.21.15.83:0/810846788 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f9ccc06fb70 0x7f9ccc127580 secure :-1 s=READY pgs=13 cs=0 l=1 rev1=1 crypto rx=0x7f9cc0011a80 tx=0x7f9cc0011f40 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:38:44.376 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.373+0000 7f9cd2501640 1 --2- 172.21.15.83:0/810846788 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f9ccc06f250 0x7f9ccc127040 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-09-14T03:38:44.376 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.374+0000 7f9cbeffd640 1 -- 172.21.15.83:0/810846788 <== mon.2 v2:172.21.15.83:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9cc000dc40 con 0x7f9ccc06fb70 2024-09-14T03:38:44.377 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.374+0000 7f9cbeffd640 1 -- 172.21.15.83:0/810846788 <== mon.2 v2:172.21.15.83:3302/0 2 ==== config(4 keys) ==== 110+0+0 (secure 0 0 0) 0x7f9cc0012450 con 0x7f9ccc06fb70 2024-09-14T03:38:44.377 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.375+0000 7f9cbeffd640 1 -- 172.21.15.83:0/810846788 <== mon.2 v2:172.21.15.83:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9cc0012800 con 0x7f9ccc06fb70 2024-09-14T03:38:44.377 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.375+0000 7f9cd4765640 1 -- 172.21.15.83:0/810846788 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f9ccc12a360 con 0x7f9ccc06fb70 2024-09-14T03:38:44.377 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.375+0000 7f9cd4765640 1 -- 172.21.15.83:0/810846788 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7f9ccc128f70 con 0x7f9ccc06fb70 2024-09-14T03:38:44.377 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.376+0000 7f9cbeffd640 1 -- 172.21.15.83:0/810846788 <== mon.2 v2:172.21.15.83:3302/0 4 ==== mgrmap(e 1) ==== 807+0+0 (secure 0 0 0) 0x7f9cc00129a0 con 0x7f9ccc06fb70 2024-09-14T03:38:44.377 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.376+0000 7f9cbeffd640 1 -- 172.21.15.83:0/810846788 <== mon.2 v2:172.21.15.83:3302/0 5 ==== osd_map(1..1 src has 1..1) ==== 725+0+0 (secure 0 0 0) 0x7f9cc001b770 con 0x7f9ccc06fb70 2024-09-14T03:38:44.381 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.379+0000 7fb7e4fc3640 1 -- 172.21.15.83:0/1078274444 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fb7e0127c70 msgr2=0x7fb7e0132da0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:38:44.381 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.379+0000 7fb7e4fc3640 1 --2- 172.21.15.83:0/1078274444 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fb7e0127c70 0x7fb7e0132da0 secure :-1 s=READY pgs=12 cs=0 l=1 rev1=1 crypto rx=0x7fb7d4009830 tx=0x7fb7d402a320 comp rx=0 tx=0).stop 2024-09-14T03:38:44.381 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.380+0000 7fb7e4fc3640 1 -- 172.21.15.83:0/1078274444 shutdown_connections 2024-09-14T03:38:44.381 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.380+0000 7fb7e4fc3640 1 --2- 172.21.15.83:0/1078274444 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fb7e0127c70 0x7fb7e0132da0 unknown :-1 s=CLOSED pgs=12 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:44.381 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.380+0000 7fb7e4fc3640 1 --2- 172.21.15.83:0/1078274444 >> v2:172.21.15.83:3300/0 conn(0x7fb7e012b6c0 0x7fb7e0092660 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:44.381 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.380+0000 7fb7e4fc3640 1 -- 172.21.15.83:0/1078274444 >> 172.21.15.83:0/1078274444 conn(0x7fb7e009d5f0 msgr2=0x7fb7e009da00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:38:44.382 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.381+0000 7f9cd4765640 1 -- 172.21.15.83:0/810846788 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f9ccc129340 con 0x7f9ccc06fb70 2024-09-14T03:38:44.384 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.382+0000 7fb7e4fc3640 1 -- 172.21.15.83:0/1078274444 shutdown_connections 2024-09-14T03:38:44.384 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.381+0000 7f99e5ad7640 1 -- 172.21.15.83:0/1227517626 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_command([{prefix=config set, name=mds_debug_frag}] v 0) -- 0x7f99c0004360 con 0x7f99e0093e70 2024-09-14T03:38:44.396 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.391+0000 7fb7e4fc3640 1 -- 172.21.15.83:0/1078274444 wait complete. 2024-09-14T03:38:44.397 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.392+0000 7fb7e4fc3640 1 Processor -- start 2024-09-14T03:38:44.397 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.392+0000 7fb7e4fc3640 1 -- start start 2024-09-14T03:38:44.397 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.393+0000 7fb7e4fc3640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fb7e012acd0 0x7fb7e0073430 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:38:44.397 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.393+0000 7fb7e4fc3640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fb7e012b6c0 0x7fb7e006eb00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:38:44.397 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.393+0000 7fb7e4fc3640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fb7e006f040 0x7fb7e006f490 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:38:44.397 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.393+0000 7fb7e4fc3640 1 -- --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7fb7e006bfc0 con 0x7fb7e012acd0 2024-09-14T03:38:44.397 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.393+0000 7fb7e4fc3640 1 -- --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7fb7e006be40 con 0x7fb7e012b6c0 2024-09-14T03:38:44.397 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.393+0000 7fb7e4fc3640 1 -- --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7fb7e006c140 con 0x7fb7e006f040 2024-09-14T03:38:44.397 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.393+0000 7fb7ddd9b640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fb7e012b6c0 0x7fb7e006eb00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:38:44.397 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.393+0000 7fb7ddd9b640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fb7e012b6c0 0x7fb7e006eb00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3301/0 says I am v2:172.21.15.83:37486/0 (socket says 172.21.15.83:37486) 2024-09-14T03:38:44.397 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.393+0000 7fb7ddd9b640 1 -- 172.21.15.83:0/3954769710 learned_addr learned my addr 172.21.15.83:0/3954769710 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:38:44.397 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.394+0000 7fb7ddd9b640 1 -- 172.21.15.83:0/3954769710 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fb7e006f040 msgr2=0x7fb7e006f490 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:38:44.398 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.394+0000 7fb7ddd9b640 1 --2- 172.21.15.83:0/3954769710 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fb7e006f040 0x7fb7e006f490 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:44.398 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.394+0000 7fb7ddd9b640 1 -- 172.21.15.83:0/3954769710 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fb7e012acd0 msgr2=0x7fb7e0073430 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:38:44.398 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.394+0000 7fb7ddd9b640 1 --2- 172.21.15.83:0/3954769710 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fb7e012acd0 0x7fb7e0073430 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:44.398 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.394+0000 7fb7ddd9b640 1 -- 172.21.15.83:0/3954769710 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fb7e00708d0 con 0x7fb7e012b6c0 2024-09-14T03:38:44.398 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.394+0000 7fb7ddd9b640 1 --2- 172.21.15.83:0/3954769710 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fb7e012b6c0 0x7fb7e006eb00 secure :-1 s=READY pgs=13 cs=0 l=1 rev1=1 crypto rx=0x7fb7d000c780 tx=0x7fb7d000cc40 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:38:44.400 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.395+0000 7f9cbeffd640 1 -- 172.21.15.83:0/810846788 <== mon.2 v2:172.21.15.83:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+74432 (secure 0 0 0) 0x7f9cc0012e20 con 0x7f9ccc06fb70 2024-09-14T03:38:44.401 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.400+0000 7f99c7fff640 1 -- 172.21.15.83:0/1227517626 <== mon.1 v2:172.21.15.83:3301/0 11 ==== mon_command_ack([{prefix=config set, name=mds_debug_frag}]=0 v9)=0 v9) ==== 115+0+0 (secure 0 0 0) 0x7f99d000f6e0 con 0x7f99e0093e70 2024-09-14T03:38:44.403 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.401+0000 7fb7ceffd640 1 -- 172.21.15.83:0/3954769710 <== mon.1 v2:172.21.15.83:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb7d0017070 con 0x7fb7e012b6c0 2024-09-14T03:38:44.403 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.401+0000 7fb7ceffd640 1 -- 172.21.15.83:0/3954769710 <== mon.1 v2:172.21.15.83:3301/0 2 ==== config(4 keys) ==== 110+0+0 (secure 0 0 0) 0x7fb7d0005360 con 0x7fb7e012b6c0 2024-09-14T03:38:44.403 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.402+0000 7fb7ceffd640 1 -- 172.21.15.83:0/3954769710 <== mon.1 v2:172.21.15.83:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb7d000a8a0 con 0x7fb7e012b6c0 2024-09-14T03:38:44.404 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.403+0000 7fb7e4fc3640 1 -- 172.21.15.83:0/3954769710 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fb7e0097250 con 0x7fb7e012b6c0 2024-09-14T03:38:44.404 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.403+0000 7fb7e4fc3640 1 -- 172.21.15.83:0/3954769710 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7fb7e0070b60 con 0x7fb7e012b6c0 2024-09-14T03:38:44.405 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.403+0000 7f99e5ad7640 1 -- 172.21.15.83:0/1227517626 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f99e0093e70 msgr2=0x7f99e006e970 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:38:44.405 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.403+0000 7f99e5ad7640 1 --2- 172.21.15.83:0/1227517626 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f99e0093e70 0x7f99e006e970 secure :-1 s=READY pgs=11 cs=0 l=1 rev1=1 crypto rx=0x7f99d00011a0 tx=0x7f99d0001660 comp rx=0 tx=0).stop 2024-09-14T03:38:44.405 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.403+0000 7f99e5ad7640 1 -- 172.21.15.83:0/1227517626 shutdown_connections 2024-09-14T03:38:44.405 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.403+0000 7f99e5ad7640 1 --2- 172.21.15.83:0/1227517626 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f99e00949a0 0x7f99e006f3f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:44.405 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.403+0000 7f99e5ad7640 1 --2- 172.21.15.83:0/1227517626 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f99e00942c0 0x7f99e006eeb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:44.406 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.403+0000 7f99e5ad7640 1 --2- 172.21.15.83:0/1227517626 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f99e0093e70 0x7f99e006e970 unknown :-1 s=CLOSED pgs=11 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:44.406 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.403+0000 7f99e5ad7640 1 -- 172.21.15.83:0/1227517626 >> 172.21.15.83:0/1227517626 conn(0x7f99e009a0c0 msgr2=0x7f99e0084410 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:38:44.406 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.403+0000 7f99e5ad7640 1 -- 172.21.15.83:0/1227517626 shutdown_connections 2024-09-14T03:38:44.406 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.404+0000 7f99e5ad7640 1 -- 172.21.15.83:0/1227517626 wait complete. 2024-09-14T03:38:44.406 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.404+0000 7fb7ceffd640 1 -- 172.21.15.83:0/3954769710 <== mon.1 v2:172.21.15.83:3301/0 4 ==== mgrmap(e 1) ==== 807+0+0 (secure 0 0 0) 0x7fb7d001a070 con 0x7fb7e012b6c0 2024-09-14T03:38:44.407 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.405+0000 7fb7e4fc3640 1 -- 172.21.15.83:0/3954769710 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fb7e0092660 con 0x7fb7e012b6c0 2024-09-14T03:38:44.408 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.406+0000 7f5d089ab640 1 -- 172.21.15.83:0/1618577224 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f5cfc00d9c0 msgr2=0x7f5cfc0d6520 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:38:44.408 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.406+0000 7f5d089ab640 1 --2- 172.21.15.83:0/1618577224 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f5cfc00d9c0 0x7f5cfc0d6520 secure :-1 s=READY pgs=23 cs=0 l=1 rev1=1 crypto rx=0x7f5d04074910 tx=0x7f5d0406ad20 comp rx=0 tx=0).stop 2024-09-14T03:38:44.408 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.407+0000 7f5d089ab640 1 -- 172.21.15.83:0/1618577224 shutdown_connections 2024-09-14T03:38:44.408 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.407+0000 7f5d089ab640 1 --2- 172.21.15.83:0/1618577224 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f5cfc0d6a60 0x7f5cfc0d8e50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:44.408 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.407+0000 7f5d089ab640 1 --2- 172.21.15.83:0/1618577224 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f5cfc00d9c0 0x7f5cfc0d6520 unknown :-1 s=CLOSED pgs=23 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:44.409 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.407+0000 7f5d089ab640 1 -- 172.21.15.83:0/1618577224 >> 172.21.15.83:0/1618577224 conn(0x7f5cfc005a10 msgr2=0x7f5cfc005e20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:38:44.409 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.408+0000 7fb7ceffd640 1 -- 172.21.15.83:0/3954769710 <== mon.1 v2:172.21.15.83:3301/0 5 ==== osd_map(1..1 src has 1..1) ==== 725+0+0 (secure 0 0 0) 0x7fb7d001c440 con 0x7fb7e012b6c0 2024-09-14T03:38:44.415 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.414+0000 7fb7ceffd640 1 -- 172.21.15.83:0/3954769710 <== mon.1 v2:172.21.15.83:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+74432 (secure 0 0 0) 0x7fb7d0021070 con 0x7fb7e012b6c0 2024-09-14T03:38:44.419 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.417+0000 7f88ac3db640 1 -- 172.21.15.83:0/662044618 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f889c01d450 msgr2=0x7f889c01d8d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:38:44.419 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.417+0000 7f88ac3db640 1 --2- 172.21.15.83:0/662044618 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f889c01d450 0x7f889c01d8d0 secure :-1 s=READY pgs=24 cs=0 l=1 rev1=1 crypto rx=0x7f88a40518b0 tx=0x7f88a406ad20 comp rx=0 tx=0).stop 2024-09-14T03:38:44.419 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.418+0000 7f88ac3db640 1 -- 172.21.15.83:0/662044618 shutdown_connections 2024-09-14T03:38:44.419 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.418+0000 7f88ac3db640 1 --2- 172.21.15.83:0/662044618 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f889c01d450 0x7f889c01d8d0 unknown :-1 s=CLOSED pgs=24 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:44.419 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.418+0000 7f88ac3db640 1 --2- 172.21.15.83:0/662044618 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f889c01cb30 0x7f889c01cf10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:44.419 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.418+0000 7f88ac3db640 1 --2- 172.21.15.83:0/662044618 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f889c01c210 0x7f889c01c5f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:44.420 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.418+0000 7f88ac3db640 1 -- 172.21.15.83:0/662044618 >> 172.21.15.83:0/662044618 conn(0x7f889c005a30 msgr2=0x7f889c005e40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:38:44.422 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.412+0000 7f5d089ab640 1 -- 172.21.15.83:0/1618577224 shutdown_connections 2024-09-14T03:38:44.422 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.412+0000 7f5d089ab640 1 -- 172.21.15.83:0/1618577224 wait complete. 2024-09-14T03:38:44.424 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.418+0000 7f88ac3db640 1 -- 172.21.15.83:0/662044618 shutdown_connections 2024-09-14T03:38:44.424 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.418+0000 7f88ac3db640 1 -- 172.21.15.83:0/662044618 wait complete. 2024-09-14T03:38:44.424 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.419+0000 7f88ac3db640 1 Processor -- start 2024-09-14T03:38:44.424 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.419+0000 7f88ac3db640 1 -- start start 2024-09-14T03:38:44.424 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.420+0000 7f88ac3db640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f889c01c210 0x7f889c166ee0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:38:44.424 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.420+0000 7f88ac3db640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f889c01cb30 0x7f889c167420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:38:44.424 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.420+0000 7f88a9976640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f889c01cb30 0x7f889c167420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:38:44.424 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.420+0000 7f88a9976640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f889c01cb30 0x7f889c167420 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3302/0 says I am v2:172.21.15.83:60308/0 (socket says 172.21.15.83:60308) 2024-09-14T03:38:44.425 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.420+0000 7f88a9976640 1 -- 172.21.15.83:0/3686614296 learned_addr learned my addr 172.21.15.83:0/3686614296 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:38:44.425 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.420+0000 7f88ac3db640 1 --2- 172.21.15.83:0/3686614296 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f889c01d450 0x7f889c167960 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:38:44.425 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.420+0000 7f88ac3db640 1 -- 172.21.15.83:0/3686614296 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7f889c0b4af0 con 0x7f889c01c210 2024-09-14T03:38:44.425 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.420+0000 7f88ac3db640 1 -- 172.21.15.83:0/3686614296 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7f889c0b4970 con 0x7f889c01d450 2024-09-14T03:38:44.425 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.420+0000 7f88ac3db640 1 -- 172.21.15.83:0/3686614296 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7f889c0b4c70 con 0x7f889c01cb30 2024-09-14T03:38:44.425 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.421+0000 7f88a9976640 1 -- 172.21.15.83:0/3686614296 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f889c01d450 msgr2=0x7f889c167960 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-09-14T03:38:44.425 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.421+0000 7f88a9976640 1 --2- 172.21.15.83:0/3686614296 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f889c01d450 0x7f889c167960 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:44.425 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.421+0000 7f88a9976640 1 -- 172.21.15.83:0/3686614296 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f889c01c210 msgr2=0x7f889c166ee0 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-09-14T03:38:44.425 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.421+0000 7f88a9976640 1 --2- 172.21.15.83:0/3686614296 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f889c01c210 0x7f889c166ee0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:44.425 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.421+0000 7f88a9976640 1 -- 172.21.15.83:0/3686614296 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f889c0a7760 con 0x7f889c01cb30 2024-09-14T03:38:44.425 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.421+0000 7f88a9976640 1 --2- 172.21.15.83:0/3686614296 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f889c01cb30 0x7f889c167420 secure :-1 s=READY pgs=14 cs=0 l=1 rev1=1 crypto rx=0x7f88a000ed60 tx=0x7f88a000c6a0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:38:44.426 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.422+0000 7f8896ffd640 1 -- 172.21.15.83:0/3686614296 <== mon.2 v2:172.21.15.83:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f88a0009e10 con 0x7f889c01cb30 2024-09-14T03:38:44.426 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.422+0000 7f8896ffd640 1 -- 172.21.15.83:0/3686614296 <== mon.2 v2:172.21.15.83:3302/0 2 ==== config(4 keys) ==== 110+0+0 (secure 0 0 0) 0x7f88a0010430 con 0x7f889c01cb30 2024-09-14T03:38:44.426 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.422+0000 7f8896ffd640 1 -- 172.21.15.83:0/3686614296 <== mon.2 v2:172.21.15.83:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f88a0010800 con 0x7f889c01cb30 2024-09-14T03:38:44.426 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.413+0000 7f5d089ab640 1 Processor -- start 2024-09-14T03:38:44.426 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.414+0000 7f5d089ab640 1 -- start start 2024-09-14T03:38:44.427 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.423+0000 7f88ac3db640 1 -- 172.21.15.83:0/3686614296 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f889c012b70 con 0x7f889c01cb30 2024-09-14T03:38:44.427 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.424+0000 7f88ac3db640 1 -- 172.21.15.83:0/3686614296 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7f889c0a6470 con 0x7f889c01cb30 2024-09-14T03:38:44.427 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.424+0000 7f8896ffd640 1 -- 172.21.15.83:0/3686614296 <== mon.2 v2:172.21.15.83:3302/0 4 ==== mgrmap(e 1) ==== 807+0+0 (secure 0 0 0) 0x7f88a00109a0 con 0x7f889c01cb30 2024-09-14T03:38:44.427 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.424+0000 7f8896ffd640 1 -- 172.21.15.83:0/3686614296 <== mon.2 v2:172.21.15.83:3302/0 5 ==== osd_map(1..1 src has 1..1) ==== 725+0+0 (secure 0 0 0) 0x7f88a00213f0 con 0x7f889c01cb30 2024-09-14T03:38:44.427 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.425+0000 7f8894ff9640 1 -- 172.21.15.83:0/3686614296 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f889c01cf10 con 0x7f889c01cb30 2024-09-14T03:38:44.431 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.414+0000 7f5d089ab640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f5cfc00ced0 0x7f5cfc015ef0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:38:44.432 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.414+0000 7f5d089ab640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f5cfc00d9c0 0x7f5cfc016430 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:38:44.432 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.414+0000 7f5d089ab640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f5cfc0d6a60 0x7f5cfc016970 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:38:44.432 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.414+0000 7f5d089ab640 1 -- --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7f5cfc0ddf00 con 0x7f5cfc0d6a60 2024-09-14T03:38:44.432 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.414+0000 7f5d089ab640 1 -- --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7f5cfc0ddd80 con 0x7f5cfc00d9c0 2024-09-14T03:38:44.432 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.414+0000 7f5d089ab640 1 -- --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7f5cfc0de080 con 0x7f5cfc00ced0 2024-09-14T03:38:44.432 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.416+0000 7f5d02ffd640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f5cfc00d9c0 0x7f5cfc016430 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:38:44.432 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.416+0000 7f5d02ffd640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f5cfc00d9c0 0x7f5cfc016430 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3301/0 says I am v2:172.21.15.83:37492/0 (socket says 172.21.15.83:37492) 2024-09-14T03:38:44.432 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.416+0000 7f5d02ffd640 1 -- 172.21.15.83:0/2801243251 learned_addr learned my addr 172.21.15.83:0/2801243251 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:38:44.432 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.416+0000 7f5d037fe640 1 --2- 172.21.15.83:0/2801243251 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f5cfc00ced0 0x7f5cfc015ef0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:38:44.432 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.416+0000 7f5d03fff640 1 --2- 172.21.15.83:0/2801243251 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f5cfc0d6a60 0x7f5cfc016970 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:38:44.432 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.419+0000 7f5d02ffd640 1 -- 172.21.15.83:0/2801243251 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f5cfc00ced0 msgr2=0x7f5cfc015ef0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:38:44.432 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.419+0000 7f5d02ffd640 1 --2- 172.21.15.83:0/2801243251 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f5cfc00ced0 0x7f5cfc015ef0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:44.433 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.419+0000 7f5d02ffd640 1 -- 172.21.15.83:0/2801243251 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f5cfc0d6a60 msgr2=0x7f5cfc016970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:38:44.433 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.419+0000 7f5d02ffd640 1 --2- 172.21.15.83:0/2801243251 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f5cfc0d6a60 0x7f5cfc016970 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:44.433 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.419+0000 7f5d02ffd640 1 -- 172.21.15.83:0/2801243251 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f5cfc0196f0 con 0x7f5cfc00d9c0 2024-09-14T03:38:44.433 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.420+0000 7f5d02ffd640 1 --2- 172.21.15.83:0/2801243251 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f5cfc00d9c0 0x7f5cfc016430 secure :-1 s=READY pgs=14 cs=0 l=1 rev1=1 crypto rx=0x7f5d04074910 tx=0x7f5d0409b5d0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:38:44.433 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.420+0000 7f5ceffff640 1 -- 172.21.15.83:0/2801243251 <== mon.1 v2:172.21.15.83:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5d040ac020 con 0x7f5cfc00d9c0 2024-09-14T03:38:44.433 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.420+0000 7f5ceffff640 1 -- 172.21.15.83:0/2801243251 <== mon.1 v2:172.21.15.83:3301/0 2 ==== config(4 keys) ==== 110+0+0 (secure 0 0 0) 0x7f5d040a2040 con 0x7f5cfc00d9c0 2024-09-14T03:38:44.433 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.420+0000 7f5ceffff640 1 -- 172.21.15.83:0/2801243251 <== mon.1 v2:172.21.15.83:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5d040a1050 con 0x7f5cfc00d9c0 2024-09-14T03:38:44.433 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.421+0000 7f5d089ab640 1 -- 172.21.15.83:0/2801243251 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f5cfc018d80 con 0x7f5cfc00d9c0 2024-09-14T03:38:44.433 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.425+0000 7f5d089ab640 1 -- 172.21.15.83:0/2801243251 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7f5cfc01e5e0 con 0x7f5cfc00d9c0 2024-09-14T03:38:44.433 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.425+0000 7f5ceffff640 1 -- 172.21.15.83:0/2801243251 <== mon.1 v2:172.21.15.83:3301/0 4 ==== mgrmap(e 1) ==== 807+0+0 (secure 0 0 0) 0x7f5d0409bd20 con 0x7f5cfc00d9c0 2024-09-14T03:38:44.434 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.428+0000 7f5ceffff640 1 -- 172.21.15.83:0/2801243251 <== mon.1 v2:172.21.15.83:3301/0 5 ==== osd_map(1..1 src has 1..1) ==== 725+0+0 (secure 0 0 0) 0x7f5d040a8b30 con 0x7f5cfc00d9c0 2024-09-14T03:38:44.434 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.429+0000 7f5d089ab640 1 -- 172.21.15.83:0/2801243251 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f5cfc00afd0 con 0x7f5cfc00d9c0 2024-09-14T03:38:44.434 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.429+0000 7f8896ffd640 1 -- 172.21.15.83:0/3686614296 <== mon.2 v2:172.21.15.83:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+74432 (secure 0 0 0) 0x7f88a0010c90 con 0x7f889c01cb30 2024-09-14T03:38:44.435 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.433+0000 7f5ceffff640 1 -- 172.21.15.83:0/2801243251 <== mon.1 v2:172.21.15.83:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+74432 (secure 0 0 0) 0x7f5cfc00afd0 con 0x7f5cfc00d9c0 2024-09-14T03:38:44.464 INFO:tasks.ceph.mgr.x.smithi083.stderr:2024-09-14T03:38:44.463+0000 7f5470a36040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-14T03:38:44.475 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.473+0000 7fe575118640 1 -- 172.21.15.83:0/2554446255 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fe570095800 msgr2=0x7fe570094e80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:38:44.475 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.473+0000 7fe575118640 1 --2- 172.21.15.83:0/2554446255 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fe570095800 0x7fe570094e80 secure :-1 s=READY pgs=15 cs=0 l=1 rev1=1 crypto rx=0x7fe56401a910 tx=0x7fe56403b490 comp rx=0 tx=0).stop 2024-09-14T03:38:44.477 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.476+0000 7fe575118640 1 -- 172.21.15.83:0/2554446255 shutdown_connections 2024-09-14T03:38:44.477 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.476+0000 7fe575118640 1 --2- 172.21.15.83:0/2554446255 >> v2:172.21.15.83:3300/0 conn(0x7fe5701369f0 0x7fe570138de0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:44.477 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.476+0000 7fe575118640 1 --2- 172.21.15.83:0/2554446255 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fe570095800 0x7fe570094e80 unknown :-1 s=CLOSED pgs=15 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:44.477 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.476+0000 7fe575118640 1 --2- 172.21.15.83:0/2554446255 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fe570094560 0x7fe570094940 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:44.477 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.476+0000 7fe575118640 1 -- 172.21.15.83:0/2554446255 >> 172.21.15.83:0/2554446255 conn(0x7fe57009c680 msgr2=0x7fe570098b50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:38:44.485 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.483+0000 7f1f3db9c640 1 -- 172.21.15.83:0/4056006539 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f1f38094aa0 msgr2=0x7f1f38094ee0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:38:44.485 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.483+0000 7f1f3db9c640 1 --2- 172.21.15.83:0/4056006539 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f1f38094aa0 0x7f1f38094ee0 secure :-1 s=READY pgs=16 cs=0 l=1 rev1=1 crypto rx=0x7f1f24009900 tx=0x7f1f2402a4d0 comp rx=0 tx=0).stop 2024-09-14T03:38:44.486 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.484+0000 7fe575118640 1 -- 172.21.15.83:0/2554446255 shutdown_connections 2024-09-14T03:38:44.486 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.484+0000 7f1f3db9c640 1 -- 172.21.15.83:0/4056006539 shutdown_connections 2024-09-14T03:38:44.486 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.484+0000 7f1f3db9c640 1 --2- 172.21.15.83:0/4056006539 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f1f38094aa0 0x7f1f38094ee0 unknown :-1 s=CLOSED pgs=16 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:44.486 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.484+0000 7f1f3db9c640 1 --2- 172.21.15.83:0/4056006539 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f1f38095800 0x7f1f38094560 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:44.486 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.484+0000 7f1f3db9c640 1 -- 172.21.15.83:0/4056006539 >> 172.21.15.83:0/4056006539 conn(0x7f1f3809c680 msgr2=0x7f1f38098b50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:38:44.488 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.486+0000 7fbbc99ff640 1 -- 172.21.15.83:0/2559165777 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fbbc4095050 msgr2=0x7fbbc412d700 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:38:44.488 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.486+0000 7fbbc99ff640 1 --2- 172.21.15.83:0/2559165777 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fbbc4095050 0x7fbbc412d700 secure :-1 s=READY pgs=15 cs=0 l=1 rev1=1 crypto rx=0x7fbbbc005fd0 tx=0x7fbbbc02abc0 comp rx=0 tx=0).stop 2024-09-14T03:38:44.488 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.487+0000 7fe575118640 1 -- 172.21.15.83:0/2554446255 wait complete. 2024-09-14T03:38:44.489 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.487+0000 7f1f3db9c640 1 -- 172.21.15.83:0/4056006539 shutdown_connections 2024-09-14T03:38:44.490 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.488+0000 7fe575118640 1 Processor -- start 2024-09-14T03:38:44.491 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.487+0000 7f1f3db9c640 1 -- 172.21.15.83:0/4056006539 wait complete. 2024-09-14T03:38:44.491 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.488+0000 7f1f3db9c640 1 Processor -- start 2024-09-14T03:38:44.491 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.488+0000 7f1f3db9c640 1 -- start start 2024-09-14T03:38:44.491 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.489+0000 7f1f3db9c640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f1f38094aa0 0x7f1f3806f040 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:38:44.491 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.489+0000 7f1f3db9c640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f1f38095800 0x7f1f3806f580 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:38:44.491 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.489+0000 7f1f3db9c640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f1f380727f0 0x7f1f38072c70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:38:44.492 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.489+0000 7f1f3db9c640 1 -- --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7f1f3813ae90 con 0x7f1f38095800 2024-09-14T03:38:44.492 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.489+0000 7f1f3db9c640 1 -- --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7f1f3813ad10 con 0x7f1f380727f0 2024-09-14T03:38:44.492 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.489+0000 7f1f3db9c640 1 -- --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7f1f3813b010 con 0x7f1f38094aa0 2024-09-14T03:38:44.492 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.489+0000 7fe575118640 1 -- start start 2024-09-14T03:38:44.492 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.489+0000 7fe575118640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fe570094560 0x7fe57012d1d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:38:44.492 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.489+0000 7fe575118640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fe5701369f0 0x7fe57012d710 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:38:44.492 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.489+0000 7fe575118640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fe570091bc0 0x7fe57012dc50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:38:44.492 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.489+0000 7fe575118640 1 -- --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7fe57012eb40 con 0x7fe570094560 2024-09-14T03:38:44.493 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.489+0000 7fe575118640 1 -- --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7fe57012e9c0 con 0x7fe5701369f0 2024-09-14T03:38:44.493 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.489+0000 7fe575118640 1 -- --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7fe57012ecc0 con 0x7fe570091bc0 2024-09-14T03:38:44.493 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.490+0000 7f1f36ffd640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f1f38095800 0x7f1f3806f580 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:38:44.493 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.490+0000 7fe56ed9d640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fe570094560 0x7fe57012d1d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:38:44.493 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.490+0000 7fe56ed9d640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fe570094560 0x7fe57012d1d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3300/0 says I am v2:172.21.15.83:43948/0 (socket says 172.21.15.83:43948) 2024-09-14T03:38:44.493 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.490+0000 7fe56f59e640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fe570091bc0 0x7fe57012dc50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:38:44.493 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.490+0000 7fe56ed9d640 1 -- 172.21.15.83:0/488579304 learned_addr learned my addr 172.21.15.83:0/488579304 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:38:44.493 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.490+0000 7fe56e59c640 1 --2- 172.21.15.83:0/488579304 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fe5701369f0 0x7fe57012d710 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:38:44.494 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.490+0000 7f1f36ffd640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f1f38095800 0x7f1f3806f580 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3300/0 says I am v2:172.21.15.83:43936/0 (socket says 172.21.15.83:43936) 2024-09-14T03:38:44.494 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.490+0000 7f1f36ffd640 1 -- 172.21.15.83:0/1198565397 learned_addr learned my addr 172.21.15.83:0/1198565397 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:38:44.494 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.490+0000 7f1f36ffd640 1 -- 172.21.15.83:0/1198565397 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f1f38094aa0 msgr2=0x7f1f3806f040 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-09-14T03:38:44.494 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.490+0000 7f1f36ffd640 1 --2- 172.21.15.83:0/1198565397 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f1f38094aa0 0x7f1f3806f040 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:44.494 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.490+0000 7f1f36ffd640 1 -- 172.21.15.83:0/1198565397 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f1f380727f0 msgr2=0x7f1f38072c70 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-09-14T03:38:44.494 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.490+0000 7f1f36ffd640 1 --2- 172.21.15.83:0/1198565397 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f1f380727f0 0x7f1f38072c70 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:44.495 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.490+0000 7fe56f59e640 1 -- 172.21.15.83:0/488579304 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fe5701369f0 msgr2=0x7fe57012d710 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:38:44.495 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.490+0000 7fe56f59e640 1 --2- 172.21.15.83:0/488579304 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fe5701369f0 0x7fe57012d710 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:44.495 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.490+0000 7f1f36ffd640 1 -- 172.21.15.83:0/1198565397 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f1f380708d0 con 0x7f1f38095800 2024-09-14T03:38:44.495 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.490+0000 7fe56f59e640 1 -- 172.21.15.83:0/488579304 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fe570094560 msgr2=0x7fe57012d1d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:38:44.496 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.490+0000 7fe56f59e640 1 --2- 172.21.15.83:0/488579304 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fe570094560 0x7fe57012d1d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:44.496 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.490+0000 7fe56f59e640 1 -- 172.21.15.83:0/488579304 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fe570092520 con 0x7fe570091bc0 2024-09-14T03:38:44.496 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.491+0000 7fe56f59e640 1 --2- 172.21.15.83:0/488579304 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fe570091bc0 0x7fe57012dc50 secure :-1 s=READY pgs=17 cs=0 l=1 rev1=1 crypto rx=0x7fe56800bf70 tx=0x7fe56800c4d0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:38:44.497 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.491+0000 7f1f36ffd640 1 --2- 172.21.15.83:0/1198565397 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f1f38095800 0x7f1f3806f580 secure :-1 s=READY pgs=26 cs=0 l=1 rev1=1 crypto rx=0x7f1f24000c00 tx=0x7f1f24003bf0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:38:44.497 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.493+0000 7f1f2ffff640 1 -- 172.21.15.83:0/1198565397 <== mon.0 v2:172.21.15.83:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1f2403f070 con 0x7f1f38095800 2024-09-14T03:38:44.497 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.491+0000 7fe5577fe640 1 -- 172.21.15.83:0/488579304 <== mon.2 v2:172.21.15.83:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe568019070 con 0x7fe570091bc0 2024-09-14T03:38:44.497 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.491+0000 7fe575118640 1 -- 172.21.15.83:0/488579304 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fe5700911b0 con 0x7fe570091bc0 2024-09-14T03:38:44.497 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.491+0000 7fe575118640 1 -- 172.21.15.83:0/488579304 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7fe57008d6f0 con 0x7fe570091bc0 2024-09-14T03:38:44.497 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.492+0000 7fe5577fe640 1 -- 172.21.15.83:0/488579304 <== mon.2 v2:172.21.15.83:3302/0 2 ==== config(4 keys) ==== 110+0+0 (secure 0 0 0) 0x7fe568014440 con 0x7fe570091bc0 2024-09-14T03:38:44.497 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.492+0000 7fe5577fe640 1 -- 172.21.15.83:0/488579304 <== mon.2 v2:172.21.15.83:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe568014870 con 0x7fe570091bc0 2024-09-14T03:38:44.497 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.493+0000 7fe5577fe640 1 -- 172.21.15.83:0/488579304 <== mon.2 v2:172.21.15.83:3302/0 4 ==== mgrmap(e 1) ==== 807+0+0 (secure 0 0 0) 0x7fe56801c070 con 0x7fe570091bc0 2024-09-14T03:38:44.497 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.493+0000 7fe5577fe640 1 -- 172.21.15.83:0/488579304 <== mon.2 v2:172.21.15.83:3302/0 5 ==== osd_map(1..1 src has 1..1) ==== 725+0+0 (secure 0 0 0) 0x7fe56801d3b0 con 0x7fe570091bc0 2024-09-14T03:38:44.498 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.488+0000 7fbbc99ff640 1 -- 172.21.15.83:0/2559165777 shutdown_connections 2024-09-14T03:38:44.498 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.488+0000 7fbbc99ff640 1 --2- 172.21.15.83:0/2559165777 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fbbc4095050 0x7fbbc412d700 unknown :-1 s=CLOSED pgs=15 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:44.498 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.488+0000 7fbbc99ff640 1 --2- 172.21.15.83:0/2559165777 >> v2:172.21.15.83:3300/0 conn(0x7fbbc4095800 0x7fbbc412d1c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:44.498 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.488+0000 7fbbc99ff640 1 -- 172.21.15.83:0/2559165777 >> 172.21.15.83:0/2559165777 conn(0x7fbbc409c680 msgr2=0x7fbbc4098b50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:38:44.498 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.493+0000 7f1f2ffff640 1 -- 172.21.15.83:0/1198565397 <== mon.0 v2:172.21.15.83:3300/0 2 ==== config(4 keys) ==== 110+0+0 (secure 0 0 0) 0x7f1f240047a0 con 0x7f1f38095800 2024-09-14T03:38:44.498 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.493+0000 7f1f2ffff640 1 -- 172.21.15.83:0/1198565397 <== mon.0 v2:172.21.15.83:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1f24035030 con 0x7f1f38095800 2024-09-14T03:38:44.498 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.493+0000 7f1f3db9c640 1 -- 172.21.15.83:0/1198565397 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f1f38071650 con 0x7f1f38095800 2024-09-14T03:38:44.498 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.494+0000 7f1f3db9c640 1 -- 172.21.15.83:0/1198565397 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f1f3806fac0 con 0x7f1f38095800 2024-09-14T03:38:44.499 INFO:tasks.ceph.mgr.y.smithi083.stderr:2024-09-14T03:38:44.495+0000 7f0c09b73040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-14T03:38:44.499 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.495+0000 7f27de1a3640 1 -- 172.21.15.83:0/1467785204 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f27d8137a30 msgr2=0x7f27d8139e20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:38:44.499 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.495+0000 7f27de1a3640 1 --2- 172.21.15.83:0/1467785204 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f27d8137a30 0x7f27d8139e20 secure :-1 s=READY pgs=16 cs=0 l=1 rev1=1 crypto rx=0x7f27d0005fd0 tx=0x7f27d002a950 comp rx=0 tx=0).stop 2024-09-14T03:38:44.499 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.495+0000 7f27de1a3640 1 -- 172.21.15.83:0/1467785204 shutdown_connections 2024-09-14T03:38:44.499 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.495+0000 7f27de1a3640 1 --2- 172.21.15.83:0/1467785204 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f27d8137a30 0x7f27d8139e20 unknown :-1 s=CLOSED pgs=16 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:44.499 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.495+0000 7f27de1a3640 1 --2- 172.21.15.83:0/1467785204 >> v2:172.21.15.83:3300/0 conn(0x7f27d80726e0 0x7f27d81374f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:44.499 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.495+0000 7f27de1a3640 1 -- 172.21.15.83:0/1467785204 >> 172.21.15.83:0/1467785204 conn(0x7f27d809e860 msgr2=0x7f27d809ec70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:38:44.499 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.496+0000 7f27de1a3640 1 -- 172.21.15.83:0/1467785204 shutdown_connections 2024-09-14T03:38:44.500 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.495+0000 7f1f2ffff640 1 -- 172.21.15.83:0/1198565397 <== mon.0 v2:172.21.15.83:3300/0 4 ==== mgrmap(e 1) ==== 807+0+0 (secure 0 0 0) 0x7f1f24042070 con 0x7f1f38095800 2024-09-14T03:38:44.500 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.495+0000 7f1f2ffff640 1 -- 172.21.15.83:0/1198565397 <== mon.0 v2:172.21.15.83:3300/0 5 ==== osd_map(1..1 src has 1..1) ==== 725+0+0 (secure 0 0 0) 0x7f1f2403b9c0 con 0x7f1f38095800 2024-09-14T03:38:44.500 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.496+0000 7f27de1a3640 1 -- 172.21.15.83:0/1467785204 wait complete. 2024-09-14T03:38:44.500 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.496+0000 7fe575118640 1 -- 172.21.15.83:0/488579304 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fe550005290 con 0x7fe570091bc0 2024-09-14T03:38:44.501 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.496+0000 7f1f3db9c640 1 -- 172.21.15.83:0/1198565397 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f1f38094ee0 con 0x7f1f38095800 2024-09-14T03:38:44.501 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.497+0000 7f27de1a3640 1 Processor -- start 2024-09-14T03:38:44.501 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.497+0000 7f27de1a3640 1 -- start start 2024-09-14T03:38:44.501 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.498+0000 7f27de1a3640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f27d806ff40 0x7f27d81c8660 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:38:44.501 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.499+0000 7f27d77fe640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f27d806ff40 0x7f27d81c8660 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:38:44.501 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.499+0000 7f27d77fe640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f27d806ff40 0x7f27d81c8660 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3301/0 says I am v2:172.21.15.83:37544/0 (socket says 172.21.15.83:37544) 2024-09-14T03:38:44.501 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.499+0000 7f27d77fe640 1 -- 172.21.15.83:0/3157470827 learned_addr learned my addr 172.21.15.83:0/3157470827 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:38:44.501 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.499+0000 7f27de1a3640 1 --2- 172.21.15.83:0/3157470827 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f27d80726e0 0x7f27d81c8ba0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:38:44.502 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.499+0000 7f27de1a3640 1 --2- 172.21.15.83:0/3157470827 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f27d8137a30 0x7f27d808b000 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:38:44.502 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.499+0000 7f27d6ffd640 1 --2- 172.21.15.83:0/3157470827 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f27d80726e0 0x7f27d81c8ba0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:38:44.502 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.499+0000 7f27d7fff640 1 --2- 172.21.15.83:0/3157470827 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f27d8137a30 0x7f27d808b000 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:38:44.502 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.499+0000 7f27de1a3640 1 -- 172.21.15.83:0/3157470827 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7f27d813ae10 con 0x7f27d80726e0 2024-09-14T03:38:44.502 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.499+0000 7f27de1a3640 1 -- 172.21.15.83:0/3157470827 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7f27d813ac90 con 0x7f27d806ff40 2024-09-14T03:38:44.502 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.499+0000 7f27de1a3640 1 -- 172.21.15.83:0/3157470827 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7f27d813af90 con 0x7f27d8137a30 2024-09-14T03:38:44.502 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.496+0000 7fbbc99ff640 1 -- 172.21.15.83:0/2559165777 shutdown_connections 2024-09-14T03:38:44.502 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.496+0000 7fbbc99ff640 1 -- 172.21.15.83:0/2559165777 wait complete. 2024-09-14T03:38:44.502 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.497+0000 7fbbc99ff640 1 Processor -- start 2024-09-14T03:38:44.503 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.500+0000 7fe5577fe640 1 -- 172.21.15.83:0/488579304 <== mon.2 v2:172.21.15.83:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+74432 (secure 0 0 0) 0x7fe568023020 con 0x7fe570091bc0 2024-09-14T03:38:44.503 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.500+0000 7f1f2ffff640 1 -- 172.21.15.83:0/1198565397 <== mon.0 v2:172.21.15.83:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+74432 (secure 0 0 0) 0x7f1f2403bc10 con 0x7f1f38095800 2024-09-14T03:38:44.503 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.500+0000 7f27d7fff640 1 -- 172.21.15.83:0/3157470827 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f27d806ff40 msgr2=0x7f27d81c8660 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:38:44.504 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.500+0000 7f27d7fff640 1 --2- 172.21.15.83:0/3157470827 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f27d806ff40 0x7f27d81c8660 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:44.504 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.500+0000 7f27d7fff640 1 -- 172.21.15.83:0/3157470827 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f27d80726e0 msgr2=0x7f27d81c8ba0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:38:44.504 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.500+0000 7f27d7fff640 1 --2- 172.21.15.83:0/3157470827 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f27d80726e0 0x7f27d81c8ba0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:44.504 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.501+0000 7f27d7fff640 1 -- 172.21.15.83:0/3157470827 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f27d808c770 con 0x7f27d8137a30 2024-09-14T03:38:44.504 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.500+0000 7fbbc99ff640 1 -- start start 2024-09-14T03:38:44.504 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.500+0000 7fbbc99ff640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fbbc4094560 0x7fbbc41c8150 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:38:44.504 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.500+0000 7fbbc99ff640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fbbc4095050 0x7fbbc41c8690 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:38:44.504 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.500+0000 7fbbc99ff640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fbbc4095800 0x7fbbc41c8bd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:38:44.504 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.500+0000 7fbbc99ff640 1 -- --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7fbbc413add0 con 0x7fbbc4094560 2024-09-14T03:38:44.504 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.500+0000 7fbbc99ff640 1 -- --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7fbbc413ac50 con 0x7fbbc4095050 2024-09-14T03:38:44.505 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.500+0000 7fbbc99ff640 1 -- --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7fbbc413af50 con 0x7fbbc4095800 2024-09-14T03:38:44.505 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.501+0000 7f27d7fff640 1 --2- 172.21.15.83:0/3157470827 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f27d8137a30 0x7f27d808b000 secure :-1 s=READY pgs=18 cs=0 l=1 rev1=1 crypto rx=0x7f27d8070bb0 tx=0x7f27d0004250 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:38:44.505 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.501+0000 7f27d6ffd640 1 --2- 172.21.15.83:0/3157470827 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f27d80726e0 0x7f27d81c8ba0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-09-14T03:38:44.505 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.501+0000 7f27d77fe640 1 --2- 172.21.15.83:0/3157470827 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f27d806ff40 0x7f27d81c8660 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-09-14T03:38:44.506 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.502+0000 7f27bffff640 1 -- 172.21.15.83:0/3157470827 <== mon.2 v2:172.21.15.83:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f27d0034050 con 0x7f27d8137a30 2024-09-14T03:38:44.506 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.502+0000 7f27bffff640 1 -- 172.21.15.83:0/3157470827 <== mon.2 v2:172.21.15.83:3302/0 2 ==== config(4 keys) ==== 110+0+0 (secure 0 0 0) 0x7f27d0004490 con 0x7f27d8137a30 2024-09-14T03:38:44.506 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.501+0000 7fbbc27fc640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fbbc4095050 0x7fbbc41c8690 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:38:44.506 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.501+0000 7fbbc27fc640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fbbc4095050 0x7fbbc41c8690 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3301/0 says I am v2:172.21.15.83:37558/0 (socket says 172.21.15.83:37558) 2024-09-14T03:38:44.506 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.501+0000 7fbbc27fc640 1 -- 172.21.15.83:0/1985422370 learned_addr learned my addr 172.21.15.83:0/1985422370 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:38:44.506 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.501+0000 7fbbc37fe640 1 --2- 172.21.15.83:0/1985422370 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fbbc4095800 0x7fbbc41c8bd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:38:44.506 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.502+0000 7fbbc2ffd640 1 --2- 172.21.15.83:0/1985422370 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fbbc4094560 0x7fbbc41c8150 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:38:44.507 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.502+0000 7f27de1a3640 1 -- 172.21.15.83:0/3157470827 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f27d808cf60 con 0x7f27d8137a30 2024-09-14T03:38:44.507 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.502+0000 7f27de1a3640 1 -- 172.21.15.83:0/3157470827 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7f27d808d8c0 con 0x7f27d8137a30 2024-09-14T03:38:44.507 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.502+0000 7f27bffff640 1 -- 172.21.15.83:0/3157470827 <== mon.2 v2:172.21.15.83:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f27d0004840 con 0x7f27d8137a30 2024-09-14T03:38:44.507 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.502+0000 7fbbc27fc640 1 -- 172.21.15.83:0/1985422370 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fbbc4095800 msgr2=0x7fbbc41c8bd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:38:44.507 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.502+0000 7fbbc27fc640 1 --2- 172.21.15.83:0/1985422370 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fbbc4095800 0x7fbbc41c8bd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:44.507 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.503+0000 7fbbc27fc640 1 -- 172.21.15.83:0/1985422370 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fbbc4094560 msgr2=0x7fbbc41c8150 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:38:44.507 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.503+0000 7fbbc27fc640 1 --2- 172.21.15.83:0/1985422370 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fbbc4094560 0x7fbbc41c8150 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:44.508 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.503+0000 7fbbc27fc640 1 -- 172.21.15.83:0/1985422370 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fbbc408ea30 con 0x7fbbc4095050 2024-09-14T03:38:44.508 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.503+0000 7fbbc27fc640 1 --2- 172.21.15.83:0/1985422370 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fbbc4095050 0x7fbbc41c8690 secure :-1 s=READY pgs=17 cs=0 l=1 rev1=1 crypto rx=0x7fbbb800e9c0 tx=0x7fbbb800ee80 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:38:44.508 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.503+0000 7f27bffff640 1 -- 172.21.15.83:0/3157470827 <== mon.2 v2:172.21.15.83:3302/0 4 ==== mgrmap(e 1) ==== 807+0+0 (secure 0 0 0) 0x7f27d0037020 con 0x7f27d8137a30 2024-09-14T03:38:44.508 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.504+0000 7f27bffff640 1 -- 172.21.15.83:0/3157470827 <== mon.2 v2:172.21.15.83:3302/0 5 ==== osd_map(1..1 src has 1..1) ==== 725+0+0 (secure 0 0 0) 0x7f27d00398d0 con 0x7f27d8137a30 2024-09-14T03:38:44.509 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.508+0000 7fbbabfff640 1 -- 172.21.15.83:0/1985422370 <== mon.1 v2:172.21.15.83:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fbbb800cde0 con 0x7fbbc4095050 2024-09-14T03:38:44.511 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.509+0000 7f27de1a3640 1 -- 172.21.15.83:0/3157470827 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f27b8005290 con 0x7f27d8137a30 2024-09-14T03:38:44.511 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.508+0000 7fbbc99ff640 1 -- 172.21.15.83:0/1985422370 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fbbc408d8e0 con 0x7fbbc4095050 2024-09-14T03:38:44.511 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.508+0000 7fbbc99ff640 1 -- 172.21.15.83:0/1985422370 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7fbbc408be30 con 0x7fbbc4095050 2024-09-14T03:38:44.515 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.513+0000 7f27bffff640 1 -- 172.21.15.83:0/3157470827 <== mon.2 v2:172.21.15.83:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+74432 (secure 0 0 0) 0x7f27d0039ad0 con 0x7f27d8137a30 2024-09-14T03:38:44.515 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.510+0000 7fbbabfff640 1 -- 172.21.15.83:0/1985422370 <== mon.1 v2:172.21.15.83:3301/0 2 ==== config(4 keys) ==== 110+0+0 (secure 0 0 0) 0x7fbbb8010500 con 0x7fbbc4095050 2024-09-14T03:38:44.516 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.510+0000 7fbbabfff640 1 -- 172.21.15.83:0/1985422370 <== mon.1 v2:172.21.15.83:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fbbb8010930 con 0x7fbbc4095050 2024-09-14T03:38:44.516 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.510+0000 7fbbabfff640 1 -- 172.21.15.83:0/1985422370 <== mon.1 v2:172.21.15.83:3301/0 4 ==== mgrmap(e 1) ==== 807+0+0 (secure 0 0 0) 0x7fbbb801f020 con 0x7fbbc4095050 2024-09-14T03:38:44.516 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.510+0000 7fbbabfff640 1 -- 172.21.15.83:0/1985422370 <== mon.1 v2:172.21.15.83:3301/0 5 ==== osd_map(1..1 src has 1..1) ==== 725+0+0 (secure 0 0 0) 0x7fbbb8004ae0 con 0x7fbbc4095050 2024-09-14T03:38:44.516 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.510+0000 7fbbc99ff640 1 -- 172.21.15.83:0/1985422370 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fbbc4094940 con 0x7fbbc4095050 2024-09-14T03:38:44.519 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.514+0000 7fbbabfff640 1 -- 172.21.15.83:0/1985422370 <== mon.1 v2:172.21.15.83:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+74432 (secure 0 0 0) 0x7fbbb8026070 con 0x7fbbc4095050 2024-09-14T03:38:44.521 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.520+0000 7f5bd4910640 1 -- 172.21.15.83:0/1038429552 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f5bc80b0070 msgr2=0x7f5bc8017db0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:38:44.521 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.520+0000 7f5bd4910640 1 --2- 172.21.15.83:0/1038429552 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f5bc80b0070 0x7f5bc8017db0 secure :-1 s=READY pgs=19 cs=0 l=1 rev1=1 crypto rx=0x7f5bd0074910 tx=0x7f5bd006a140 comp rx=0 tx=0).stop 2024-09-14T03:38:44.522 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.521+0000 7f5bd4910640 1 -- 172.21.15.83:0/1038429552 shutdown_connections 2024-09-14T03:38:44.522 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.521+0000 7f5bd4910640 1 --2- 172.21.15.83:0/1038429552 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f5bc80182f0 0x7f5bc8018780 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:44.522 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.521+0000 7f5bd4910640 1 --2- 172.21.15.83:0/1038429552 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f5bc80b0070 0x7f5bc8017db0 unknown :-1 s=CLOSED pgs=19 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:44.522 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.521+0000 7f5bd4910640 1 --2- 172.21.15.83:0/1038429552 >> v2:172.21.15.83:3300/0 conn(0x7f5bc80af750 0x7f5bc80afb30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:44.522 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.521+0000 7f5bd4910640 1 -- 172.21.15.83:0/1038429552 >> 172.21.15.83:0/1038429552 conn(0x7f5bc8005c20 msgr2=0x7f5bc8006030 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:38:44.524 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.523+0000 7f5bd4910640 1 -- 172.21.15.83:0/1038429552 shutdown_connections 2024-09-14T03:38:44.529 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.525+0000 7f5bd4910640 1 -- 172.21.15.83:0/1038429552 wait complete. 2024-09-14T03:38:44.530 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.527+0000 7f5bd4910640 1 Processor -- start 2024-09-14T03:38:44.530 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.527+0000 7f5bd4910640 1 -- start start 2024-09-14T03:38:44.530 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.528+0000 7f5bd4910640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f5bc80182f0 0x7f5bc80a75d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:38:44.530 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.528+0000 7f5bd4910640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f5bc80af750 0x7f5bc80a7b10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:38:44.530 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.528+0000 7f5bd4910640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f5bc80b0070 0x7f5bc80a8050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:38:44.530 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.528+0000 7f5bd4910640 1 -- --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7f5bc80b69b0 con 0x7f5bc80182f0 2024-09-14T03:38:44.530 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.528+0000 7f5bd4910640 1 -- --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7f5bc8018f80 con 0x7f5bc80af750 2024-09-14T03:38:44.530 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.528+0000 7f5bd4910640 1 -- --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7f5bc80b6b30 con 0x7f5bc80b0070 2024-09-14T03:38:44.530 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.528+0000 7f5bcf59e640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f5bc80b0070 0x7f5bc80a8050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:38:44.530 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.528+0000 7f5bced9d640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f5bc80182f0 0x7f5bc80a75d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:38:44.531 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.528+0000 7f5bced9d640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f5bc80182f0 0x7f5bc80a75d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3300/0 says I am v2:172.21.15.83:43984/0 (socket says 172.21.15.83:43984) 2024-09-14T03:38:44.531 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.528+0000 7f5bcf59e640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f5bc80b0070 0x7f5bc80a8050 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3302/0 says I am v2:172.21.15.83:60380/0 (socket says 172.21.15.83:60380) 2024-09-14T03:38:44.531 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.528+0000 7f5bcf59e640 1 -- 172.21.15.83:0/4089121259 learned_addr learned my addr 172.21.15.83:0/4089121259 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:38:44.531 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.529+0000 7f5bce59c640 1 --2- 172.21.15.83:0/4089121259 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f5bc80af750 0x7f5bc80a7b10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:38:44.531 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.529+0000 7f5bcf59e640 1 -- 172.21.15.83:0/4089121259 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f5bc80af750 msgr2=0x7f5bc80a7b10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:38:44.531 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.529+0000 7f5bcf59e640 1 --2- 172.21.15.83:0/4089121259 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f5bc80af750 0x7f5bc80a7b10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:44.532 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.529+0000 7f5bcf59e640 1 -- 172.21.15.83:0/4089121259 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f5bc80182f0 msgr2=0x7f5bc80a75d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:38:44.532 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.529+0000 7f5bcf59e640 1 --2- 172.21.15.83:0/4089121259 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f5bc80182f0 0x7f5bc80a75d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:44.532 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.529+0000 7f5bcf59e640 1 -- 172.21.15.83:0/4089121259 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f5bc801e5d0 con 0x7f5bc80b0070 2024-09-14T03:38:44.532 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.529+0000 7f5bce59c640 1 --2- 172.21.15.83:0/4089121259 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f5bc80af750 0x7f5bc80a7b10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-09-14T03:38:44.532 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.530+0000 7f5bced9d640 1 --2- 172.21.15.83:0/4089121259 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f5bc80182f0 0x7f5bc80a75d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:38:44.533 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.530+0000 7f5bcf59e640 1 --2- 172.21.15.83:0/4089121259 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f5bc80b0070 0x7f5bc80a8050 secure :-1 s=READY pgs=20 cs=0 l=1 rev1=1 crypto rx=0x7f5bc400d660 tx=0x7f5bc400db20 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:38:44.537 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.536+0000 7f5bb77fe640 1 -- 172.21.15.83:0/4089121259 <== mon.2 v2:172.21.15.83:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5bc4017070 con 0x7f5bc80b0070 2024-09-14T03:38:44.537 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.536+0000 7f5bb77fe640 1 -- 172.21.15.83:0/4089121259 <== mon.2 v2:172.21.15.83:3302/0 2 ==== config(4 keys) ==== 110+0+0 (secure 0 0 0) 0x7f5bc40052f0 con 0x7f5bc80b0070 2024-09-14T03:38:44.537 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.536+0000 7f5bd4910640 1 -- 172.21.15.83:0/4089121259 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f5bc80aa220 con 0x7f5bc80b0070 2024-09-14T03:38:44.537 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.536+0000 7f5bd4910640 1 -- 172.21.15.83:0/4089121259 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7f5bc8144fd0 con 0x7f5bc80b0070 2024-09-14T03:38:44.538 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.537+0000 7f5bb77fe640 1 -- 172.21.15.83:0/4089121259 <== mon.2 v2:172.21.15.83:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5bc4009820 con 0x7f5bc80b0070 2024-09-14T03:38:44.542 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.541+0000 7f5bd4910640 1 -- 172.21.15.83:0/4089121259 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f5bb0005290 con 0x7f5bc80b0070 2024-09-14T03:38:44.550 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.549+0000 7f5bb77fe640 1 -- 172.21.15.83:0/4089121259 <== mon.2 v2:172.21.15.83:3302/0 4 ==== mgrmap(e 1) ==== 807+0+0 (secure 0 0 0) 0x7f5bc40099c0 con 0x7f5bc80b0070 2024-09-14T03:38:44.550 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.549+0000 7f5bb77fe640 1 -- 172.21.15.83:0/4089121259 <== mon.2 v2:172.21.15.83:3302/0 5 ==== osd_map(1..1 src has 1..1) ==== 725+0+0 (secure 0 0 0) 0x7f5bc4025560 con 0x7f5bc80b0070 2024-09-14T03:38:44.550 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.549+0000 7f5bb77fe640 1 -- 172.21.15.83:0/4089121259 <== mon.2 v2:172.21.15.83:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+74432 (secure 0 0 0) 0x7f5bc4025890 con 0x7f5bc80b0070 2024-09-14T03:38:44.571 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.569+0000 7f9cd4765640 1 -- 172.21.15.83:0/810846788 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_command([{prefix=config set, name=client_mount_timeout}] v 0) -- 0x7f9ccc075950 con 0x7f9ccc06fb70 2024-09-14T03:38:44.575 INFO:tasks.ceph.mgr.y.smithi083.stderr:2024-09-14T03:38:44.574+0000 7f0c09b73040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-14T03:38:44.579 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.577+0000 7fd4deffd640 1 -- 172.21.15.83:0/1427676856 <== mon.0 v2:172.21.15.83:3300/0 7 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fd4e4012400 con 0x7fd4e8093d90 2024-09-14T03:38:44.579 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.577+0000 7f1f2ffff640 1 -- 172.21.15.83:0/1198565397 <== mon.0 v2:172.21.15.83:3300/0 7 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f1f2403bdf0 con 0x7f1f38095800 2024-09-14T03:38:44.581 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.580+0000 7fb7ceffd640 1 -- 172.21.15.83:0/3954769710 <== mon.1 v2:172.21.15.83:3301/0 7 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fb7d0003720 con 0x7fb7e012b6c0 2024-09-14T03:38:44.581 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.580+0000 7f5ceffff640 1 -- 172.21.15.83:0/2801243251 <== mon.1 v2:172.21.15.83:3301/0 7 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f5d04073040 con 0x7f5cfc00d9c0 2024-09-14T03:38:44.584 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.582+0000 7f27bffff640 1 -- 172.21.15.83:0/3157470827 <== mon.2 v2:172.21.15.83:3302/0 7 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f27d0039cb0 con 0x7f27d8137a30 2024-09-14T03:38:44.584 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.582+0000 7f5bb77fe640 1 -- 172.21.15.83:0/4089121259 <== mon.2 v2:172.21.15.83:3302/0 7 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f5bc4009cb0 con 0x7f5bc80b0070 2024-09-14T03:38:44.585 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.583+0000 7f9cbeffd640 1 -- 172.21.15.83:0/810846788 <== mon.2 v2:172.21.15.83:3302/0 7 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f9cc0024da0 con 0x7f9ccc06fb70 2024-09-14T03:38:44.585 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.583+0000 7f9cbeffd640 1 -- 172.21.15.83:0/810846788 <== mon.2 v2:172.21.15.83:3302/0 8 ==== mon_command_ack([{prefix=config set, name=client_mount_timeout}]=0 v10)=0 v10) ==== 123+0+0 (secure 0 0 0) 0x7f9cc001b9c0 con 0x7f9ccc06fb70 2024-09-14T03:38:44.585 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.584+0000 7fe5577fe640 1 -- 172.21.15.83:0/488579304 <== mon.2 v2:172.21.15.83:3302/0 7 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fe568014c70 con 0x7fe570091bc0 2024-09-14T03:38:44.586 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.581+0000 7fbbabfff640 1 -- 172.21.15.83:0/1985422370 <== mon.1 v2:172.21.15.83:3301/0 7 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fbbb8004d30 con 0x7fbbc4095050 2024-09-14T03:38:44.586 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.584+0000 7f8896ffd640 1 -- 172.21.15.83:0/3686614296 <== mon.2 v2:172.21.15.83:3302/0 7 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f88a0018420 con 0x7f889c01cb30 2024-09-14T03:38:44.586 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.584+0000 7f9cd4765640 1 -- 172.21.15.83:0/810846788 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f9ccc06fb70 msgr2=0x7f9ccc127580 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:38:44.586 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.584+0000 7f9cd4765640 1 --2- 172.21.15.83:0/810846788 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f9ccc06fb70 0x7f9ccc127580 secure :-1 s=READY pgs=13 cs=0 l=1 rev1=1 crypto rx=0x7f9cc0011a80 tx=0x7f9cc0011f40 comp rx=0 tx=0).stop 2024-09-14T03:38:44.586 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.584+0000 7f9cd4765640 1 -- 172.21.15.83:0/810846788 shutdown_connections 2024-09-14T03:38:44.587 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.584+0000 7f9cd4765640 1 --2- 172.21.15.83:0/810846788 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f9ccc0965f0 0x7f9ccc127ac0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:44.587 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.584+0000 7f9cd4765640 1 --2- 172.21.15.83:0/810846788 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f9ccc06fb70 0x7f9ccc127580 unknown :-1 s=CLOSED pgs=13 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:44.587 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.584+0000 7f9cd4765640 1 --2- 172.21.15.83:0/810846788 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f9ccc06f250 0x7f9ccc127040 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:44.587 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.584+0000 7f9cd4765640 1 -- 172.21.15.83:0/810846788 >> 172.21.15.83:0/810846788 conn(0x7f9ccc09d070 msgr2=0x7f9ccc130590 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:38:44.589 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.588+0000 7f9cd4765640 1 -- 172.21.15.83:0/810846788 shutdown_connections 2024-09-14T03:38:44.589 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.588+0000 7f9cd4765640 1 -- 172.21.15.83:0/810846788 wait complete. 2024-09-14T03:38:44.593 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.592+0000 7fb7e4fc3640 1 -- 172.21.15.83:0/3954769710 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_command([{prefix=config set, name=mds_op_complaint_time}] v 0) -- 0x7fb7e01239a0 con 0x7fb7e012b6c0 2024-09-14T03:38:44.603 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.601+0000 7fb7ceffd640 1 -- 172.21.15.83:0/3954769710 <== mon.1 v2:172.21.15.83:3301/0 8 ==== mon_command_ack([{prefix=config set, name=mds_op_complaint_time}]=0 v11)=0 v11) ==== 121+0+0 (secure 0 0 0) 0x7fb7e00708d0 con 0x7fb7e012b6c0 2024-09-14T03:38:44.604 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.602+0000 7fb7ccff9640 1 -- 172.21.15.83:0/3954769710 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fb7e012b6c0 msgr2=0x7fb7e006eb00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:38:44.604 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.602+0000 7fb7ccff9640 1 --2- 172.21.15.83:0/3954769710 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fb7e012b6c0 0x7fb7e006eb00 secure :-1 s=READY pgs=13 cs=0 l=1 rev1=1 crypto rx=0x7fb7d000c780 tx=0x7fb7d000cc40 comp rx=0 tx=0).stop 2024-09-14T03:38:44.604 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.603+0000 7fb7ccff9640 1 -- 172.21.15.83:0/3954769710 shutdown_connections 2024-09-14T03:38:44.605 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.603+0000 7fb7ccff9640 1 --2- 172.21.15.83:0/3954769710 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fb7e006f040 0x7fb7e006f490 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:44.605 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.603+0000 7fb7ccff9640 1 --2- 172.21.15.83:0/3954769710 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fb7e012b6c0 0x7fb7e006eb00 unknown :-1 s=CLOSED pgs=13 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:44.605 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.603+0000 7fb7ccff9640 1 --2- 172.21.15.83:0/3954769710 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fb7e012acd0 0x7fb7e0073430 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:44.605 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.603+0000 7fb7ccff9640 1 -- 172.21.15.83:0/3954769710 >> 172.21.15.83:0/3954769710 conn(0x7fb7e009d5f0 msgr2=0x7fb7e00985a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:38:44.605 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.603+0000 7fb7ccff9640 1 -- 172.21.15.83:0/3954769710 shutdown_connections 2024-09-14T03:38:44.605 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.603+0000 7fb7ccff9640 1 -- 172.21.15.83:0/3954769710 wait complete. 2024-09-14T03:38:44.605 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.603+0000 7f5d089ab640 1 -- 172.21.15.83:0/2801243251 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_command([{prefix=config set, name=mds_verify_scatter}] v 0) -- 0x7f5cfc008180 con 0x7f5cfc00d9c0 2024-09-14T03:38:44.612 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.610+0000 7f5ceffff640 1 -- 172.21.15.83:0/2801243251 <== mon.1 v2:172.21.15.83:3301/0 8 ==== mon_command_ack([{prefix=config set, name=mds_verify_scatter}]=0 v12)=0 v12) ==== 119+0+0 (secure 0 0 0) 0x7f5d040b6070 con 0x7f5cfc00d9c0 2024-09-14T03:38:44.618 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.616+0000 7fd4ef47e640 1 -- 172.21.15.83:0/1427676856 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_command([{prefix=config set, name=mds_debug_scatterstat}] v 0) -- 0x7fd4d0005580 con 0x7fd4e8093d90 2024-09-14T03:38:44.619 DEBUG:tasks.ceph:waiting for RemoteProcess(client=, args=['sudo', 'ceph', '--cluster', 'ceph', 'config', 'set', 'client', 'debug_client', '20'], check_status=True, hostname='smithi083') 2024-09-14T03:38:44.619 DEBUG:tasks.ceph:waiting for RemoteProcess(client=, args=['sudo', 'ceph', '--cluster', 'ceph', 'config', 'set', 'client', 'debug_ms', '1'], check_status=True, hostname='smithi083') 2024-09-14T03:38:44.619 DEBUG:tasks.ceph:waiting for RemoteProcess(client=, args=['sudo', 'ceph', '--cluster', 'ceph', 'config', 'set', 'client', 'rados_mon_op_timeout', '900'], check_status=True, hostname='smithi083') 2024-09-14T03:38:44.619 DEBUG:tasks.ceph:waiting for RemoteProcess(client=, args=['sudo', 'ceph', '--cluster', 'ceph', 'config', 'set', 'client', 'rados_osd_op_timeout', '900'], check_status=True, hostname='smithi083') 2024-09-14T03:38:44.620 DEBUG:tasks.ceph:waiting for RemoteProcess(client=, args=['sudo', 'ceph', '--cluster', 'ceph', 'config', 'set', 'mds', 'debug_mds', '20'], check_status=True, hostname='smithi083') 2024-09-14T03:38:44.620 DEBUG:tasks.ceph:waiting for RemoteProcess(client=, args=['sudo', 'ceph', '--cluster', 'ceph', 'config', 'set', 'mds', 'debug_mds_balancer', '20'], check_status=True, hostname='smithi083') 2024-09-14T03:38:44.620 DEBUG:tasks.ceph:waiting for RemoteProcess(client=, args=['sudo', 'ceph', '--cluster', 'ceph', 'config', 'set', 'mds', 'debug_ms', '2'], check_status=True, hostname='smithi083') 2024-09-14T03:38:44.620 DEBUG:tasks.ceph:waiting for RemoteProcess(client=, args=['sudo', 'ceph', '--cluster', 'ceph', 'config', 'set', 'mds', 'mds_debug_frag', 'True'], check_status=True, hostname='smithi083') 2024-09-14T03:38:44.620 DEBUG:tasks.ceph:waiting for RemoteProcess(client=, args=['sudo', 'ceph', '--cluster', 'ceph', 'config', 'set', 'mds', 'mds_debug_scatterstat', 'True'], check_status=True, hostname='smithi083') 2024-09-14T03:38:44.620 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.619+0000 7f5d089ab640 1 -- 172.21.15.83:0/2801243251 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f5cfc00d9c0 msgr2=0x7f5cfc016430 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:38:44.621 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.619+0000 7f5d089ab640 1 --2- 172.21.15.83:0/2801243251 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f5cfc00d9c0 0x7f5cfc016430 secure :-1 s=READY pgs=14 cs=0 l=1 rev1=1 crypto rx=0x7f5d04074910 tx=0x7f5d0409b5d0 comp rx=0 tx=0).stop 2024-09-14T03:38:44.621 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.619+0000 7f5d089ab640 1 -- 172.21.15.83:0/2801243251 shutdown_connections 2024-09-14T03:38:44.621 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.619+0000 7f5d089ab640 1 --2- 172.21.15.83:0/2801243251 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f5cfc0d6a60 0x7f5cfc016970 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:44.621 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.619+0000 7f5d089ab640 1 --2- 172.21.15.83:0/2801243251 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f5cfc00d9c0 0x7f5cfc016430 unknown :-1 s=CLOSED pgs=14 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:44.621 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.619+0000 7f5d089ab640 1 --2- 172.21.15.83:0/2801243251 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f5cfc00ced0 0x7f5cfc015ef0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:44.621 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.619+0000 7f5d089ab640 1 -- 172.21.15.83:0/2801243251 >> 172.21.15.83:0/2801243251 conn(0x7f5cfc005a10 msgr2=0x7f5cfc011ca0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:38:44.621 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.619+0000 7f5d089ab640 1 -- 172.21.15.83:0/2801243251 shutdown_connections 2024-09-14T03:38:44.621 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.619+0000 7f5d089ab640 1 -- 172.21.15.83:0/2801243251 wait complete. 2024-09-14T03:38:44.622 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.620+0000 7fd4deffd640 1 -- 172.21.15.83:0/1427676856 <== mon.0 v2:172.21.15.83:3300/0 8 ==== mon_command_ack([{prefix=config set, name=mds_debug_scatterstat}]=0 v13)=0 v13) ==== 122+0+0 (secure 0 0 0) 0x7fd4e40039d0 con 0x7fd4e8093d90 2024-09-14T03:38:44.628 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.623+0000 7fd4ef47e640 1 -- 172.21.15.83:0/1427676856 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fd4e8093d90 msgr2=0x7fd4e808c940 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:38:44.628 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.623+0000 7fd4ef47e640 1 --2- 172.21.15.83:0/1427676856 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fd4e8093d90 0x7fd4e808c940 secure :-1 s=READY pgs=19 cs=0 l=1 rev1=1 crypto rx=0x7fd4e400c870 tx=0x7fd4e400cd30 comp rx=0 tx=0).stop 2024-09-14T03:38:44.628 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.625+0000 7fd4ef47e640 1 -- 172.21.15.83:0/1427676856 shutdown_connections 2024-09-14T03:38:44.628 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.625+0000 7fd4ef47e640 1 --2- 172.21.15.83:0/1427676856 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fd4e8137990 0x7fd4e808ce80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:44.628 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.626+0000 7fd4ef47e640 1 --2- 172.21.15.83:0/1427676856 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fd4e8093d90 0x7fd4e808c940 unknown :-1 s=CLOSED pgs=19 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:44.628 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.626+0000 7fd4ef47e640 1 --2- 172.21.15.83:0/1427676856 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fd4e8093470 0x7fd4e808c400 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:44.628 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.626+0000 7fd4ef47e640 1 -- 172.21.15.83:0/1427676856 >> 172.21.15.83:0/1427676856 conn(0x7fd4e809c1c0 msgr2=0x7fd4e8139420 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:38:44.628 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.626+0000 7fd4ef47e640 1 -- 172.21.15.83:0/1427676856 shutdown_connections 2024-09-14T03:38:44.628 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.626+0000 7fd4ef47e640 1 -- 172.21.15.83:0/1427676856 wait complete. 2024-09-14T03:38:44.628 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.625+0000 7f8894ff9640 1 -- 172.21.15.83:0/3686614296 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_command([{prefix=config set, name=rados_mon_op_timeout}] v 0) -- 0x7f889c017ba0 con 0x7f889c01cb30 2024-09-14T03:38:44.639 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.632+0000 7f8896ffd640 1 -- 172.21.15.83:0/3686614296 <== mon.2 v2:172.21.15.83:3302/0 8 ==== mon_command_ack([{prefix=config set, name=rados_mon_op_timeout}]=0 v14)=0 v14) ==== 120+0+0 (secure 0 0 0) 0x7f88a00185d0 con 0x7f889c01cb30 2024-09-14T03:38:44.642 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.641+0000 7f8894ff9640 1 -- 172.21.15.83:0/3686614296 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f889c01cb30 msgr2=0x7f889c167420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:38:44.642 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.641+0000 7f8894ff9640 1 --2- 172.21.15.83:0/3686614296 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f889c01cb30 0x7f889c167420 secure :-1 s=READY pgs=14 cs=0 l=1 rev1=1 crypto rx=0x7f88a000ed60 tx=0x7f88a000c6a0 comp rx=0 tx=0).stop 2024-09-14T03:38:44.643 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.642+0000 7f8894ff9640 1 -- 172.21.15.83:0/3686614296 shutdown_connections 2024-09-14T03:38:44.643 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.642+0000 7f8894ff9640 1 --2- 172.21.15.83:0/3686614296 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f889c01d450 0x7f889c167960 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:44.643 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.642+0000 7f8894ff9640 1 --2- 172.21.15.83:0/3686614296 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f889c01cb30 0x7f889c167420 unknown :-1 s=CLOSED pgs=14 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:44.643 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.642+0000 7f8894ff9640 1 --2- 172.21.15.83:0/3686614296 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f889c01c210 0x7f889c166ee0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:44.643 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.642+0000 7f8894ff9640 1 -- 172.21.15.83:0/3686614296 >> 172.21.15.83:0/3686614296 conn(0x7f889c005a30 msgr2=0x7f889c016340 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:38:44.643 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.642+0000 7f8894ff9640 1 -- 172.21.15.83:0/3686614296 shutdown_connections 2024-09-14T03:38:44.644 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.642+0000 7f8894ff9640 1 -- 172.21.15.83:0/3686614296 wait complete. 2024-09-14T03:38:44.646 DEBUG:tasks.ceph:waiting for RemoteProcess(client=, args=['sudo', 'ceph', '--cluster', 'ceph', 'config', 'set', 'mds', 'mds_op_complaint_time', '180'], check_status=True, hostname='smithi083') 2024-09-14T03:38:44.646 DEBUG:tasks.ceph:waiting for RemoteProcess(client=, args=['sudo', 'ceph', '--cluster', 'ceph', 'config', 'set', 'mds', 'mds_verify_scatter', 'True'], check_status=True, hostname='smithi083') 2024-09-14T03:38:44.648 DEBUG:tasks.ceph:waiting for RemoteProcess(client=, args=['sudo', 'ceph', '--cluster', 'ceph', 'config', 'set', 'mds', 'osd_op_complaint_time', '180'], check_status=True, hostname='smithi083') 2024-09-14T03:38:44.660 INFO:tasks.ceph.mgr.y.smithi083.stderr:2024-09-14T03:38:44.658+0000 7f0c09b73040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-14T03:38:44.689 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.687+0000 7fe575118640 1 -- 172.21.15.83:0/488579304 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_command([{prefix=config set, name=rados_osd_op_timeout}] v 0) -- 0x7fe550005580 con 0x7fe570091bc0 2024-09-14T03:38:44.690 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.689+0000 7fbbc99ff640 1 -- 172.21.15.83:0/1985422370 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_command([{prefix=config set, name=mon_warn_on_pool_no_app}] v 0) -- 0x7fbbc4074000 con 0x7fbbc4095050 2024-09-14T03:38:44.696 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.695+0000 7fe5577fe640 1 -- 172.21.15.83:0/488579304 <== mon.2 v2:172.21.15.83:3302/0 8 ==== mon_command_ack([{prefix=config set, name=rados_osd_op_timeout}]=0 v15)=0 v15) ==== 120+0+0 (secure 0 0 0) 0x7fe568014e20 con 0x7fe570091bc0 2024-09-14T03:38:44.698 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.696+0000 7fe575118640 1 -- 172.21.15.83:0/488579304 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fe570091bc0 msgr2=0x7fe57012dc50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:38:44.698 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.696+0000 7fe575118640 1 --2- 172.21.15.83:0/488579304 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fe570091bc0 0x7fe57012dc50 secure :-1 s=READY pgs=17 cs=0 l=1 rev1=1 crypto rx=0x7fe56800bf70 tx=0x7fe56800c4d0 comp rx=0 tx=0).stop 2024-09-14T03:38:44.698 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.696+0000 7fe575118640 1 -- 172.21.15.83:0/488579304 shutdown_connections 2024-09-14T03:38:44.698 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.696+0000 7fe575118640 1 --2- 172.21.15.83:0/488579304 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fe570091bc0 0x7fe57012dc50 unknown :-1 s=CLOSED pgs=17 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:44.698 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.696+0000 7fe575118640 1 --2- 172.21.15.83:0/488579304 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fe5701369f0 0x7fe57012d710 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:44.698 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.696+0000 7fe575118640 1 --2- 172.21.15.83:0/488579304 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fe570094560 0x7fe57012d1d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:44.698 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.696+0000 7fe575118640 1 -- 172.21.15.83:0/488579304 >> 172.21.15.83:0/488579304 conn(0x7fe57009c680 msgr2=0x7fe570096be0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:38:44.698 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.696+0000 7fe575118640 1 -- 172.21.15.83:0/488579304 shutdown_connections 2024-09-14T03:38:44.698 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.696+0000 7fe575118640 1 -- 172.21.15.83:0/488579304 wait complete. 2024-09-14T03:38:44.705 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.703+0000 7fbbabfff640 1 -- 172.21.15.83:0/1985422370 <== mon.1 v2:172.21.15.83:3301/0 8 ==== mon_command_ack([{prefix=config set, name=mon_warn_on_pool_no_app}]=0 v16)=0 v16) ==== 125+0+0 (secure 0 0 0) 0x7fbbb8004ee0 con 0x7fbbc4095050 2024-09-14T03:38:44.707 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.705+0000 7fbbc99ff640 1 -- 172.21.15.83:0/1985422370 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fbbc4095050 msgr2=0x7fbbc41c8690 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:38:44.707 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.705+0000 7fbbc99ff640 1 --2- 172.21.15.83:0/1985422370 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fbbc4095050 0x7fbbc41c8690 secure :-1 s=READY pgs=17 cs=0 l=1 rev1=1 crypto rx=0x7fbbb800e9c0 tx=0x7fbbb800ee80 comp rx=0 tx=0).stop 2024-09-14T03:38:44.707 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.705+0000 7fbbc99ff640 1 -- 172.21.15.83:0/1985422370 shutdown_connections 2024-09-14T03:38:44.707 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.705+0000 7fbbc99ff640 1 --2- 172.21.15.83:0/1985422370 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fbbc4095800 0x7fbbc41c8bd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:44.707 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.705+0000 7fbbc99ff640 1 --2- 172.21.15.83:0/1985422370 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fbbc4095050 0x7fbbc41c8690 unknown :-1 s=CLOSED pgs=17 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:44.707 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.705+0000 7fbbc99ff640 1 --2- 172.21.15.83:0/1985422370 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fbbc4094560 0x7fbbc41c8150 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:44.707 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.705+0000 7fbbc99ff640 1 -- 172.21.15.83:0/1985422370 >> 172.21.15.83:0/1985422370 conn(0x7fbbc409c680 msgr2=0x7fbbc406d970 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:38:44.707 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.705+0000 7fbbc99ff640 1 -- 172.21.15.83:0/1985422370 shutdown_connections 2024-09-14T03:38:44.707 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.705+0000 7fbbc99ff640 1 -- 172.21.15.83:0/1985422370 wait complete. 2024-09-14T03:38:44.709 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.707+0000 7f27de1a3640 1 -- 172.21.15.83:0/3157470827 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_command([{prefix=config set, name=osd_op_complaint_time}] v 0) -- 0x7f27b8005850 con 0x7f27d8137a30 2024-09-14T03:38:44.713 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.711+0000 7f5bd4910640 1 -- 172.21.15.83:0/4089121259 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_command([{prefix=config set, name=mon_op_complaint_time}] v 0) -- 0x7f5bb0005580 con 0x7f5bc80b0070 2024-09-14T03:38:44.716 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.715+0000 7f27bffff640 1 -- 172.21.15.83:0/3157470827 <== mon.2 v2:172.21.15.83:3302/0 8 ==== mon_command_ack([{prefix=config set, name=osd_op_complaint_time}]=0 v17)=0 v17) ==== 121+0+0 (secure 0 0 0) 0x7f27d0030810 con 0x7f27d8137a30 2024-09-14T03:38:44.718 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.717+0000 7f27de1a3640 1 -- 172.21.15.83:0/3157470827 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f27d8137a30 msgr2=0x7f27d808b000 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:38:44.719 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.717+0000 7f27de1a3640 1 --2- 172.21.15.83:0/3157470827 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f27d8137a30 0x7f27d808b000 secure :-1 s=READY pgs=18 cs=0 l=1 rev1=1 crypto rx=0x7f27d8070bb0 tx=0x7f27d0004250 comp rx=0 tx=0).stop 2024-09-14T03:38:44.719 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.717+0000 7f27de1a3640 1 -- 172.21.15.83:0/3157470827 shutdown_connections 2024-09-14T03:38:44.719 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.717+0000 7f27de1a3640 1 --2- 172.21.15.83:0/3157470827 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f27d8137a30 0x7f27d808b000 unknown :-1 s=CLOSED pgs=18 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:44.719 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.717+0000 7f27de1a3640 1 --2- 172.21.15.83:0/3157470827 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f27d80726e0 0x7f27d81c8ba0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:44.719 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.717+0000 7f27de1a3640 1 --2- 172.21.15.83:0/3157470827 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f27d806ff40 0x7f27d81c8660 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:44.719 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.717+0000 7f27de1a3640 1 -- 172.21.15.83:0/3157470827 >> 172.21.15.83:0/3157470827 conn(0x7f27d809e860 msgr2=0x7f27d806dd20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:38:44.719 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.717+0000 7f27de1a3640 1 -- 172.21.15.83:0/3157470827 shutdown_connections 2024-09-14T03:38:44.719 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.717+0000 7f27de1a3640 1 -- 172.21.15.83:0/3157470827 wait complete. 2024-09-14T03:38:44.722 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.721+0000 7f5bb77fe640 1 -- 172.21.15.83:0/4089121259 <== mon.2 v2:172.21.15.83:3302/0 8 ==== mon_command_ack([{prefix=config set, name=mon_op_complaint_time}]=0 v18)=0 v18) ==== 121+0+0 (secure 0 0 0) 0x7f5bc4025a70 con 0x7f5bc80b0070 2024-09-14T03:38:44.722 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.721+0000 7f1f3db9c640 1 -- 172.21.15.83:0/1198565397 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_command([{prefix=config set, name=osd_op_complaint_time}] v 0) -- 0x7f1f38072590 con 0x7f1f38095800 2024-09-14T03:38:44.724 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.722+0000 7f5bd4910640 1 -- 172.21.15.83:0/4089121259 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f5bc80b0070 msgr2=0x7f5bc80a8050 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:38:44.724 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.722+0000 7f5bd4910640 1 --2- 172.21.15.83:0/4089121259 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f5bc80b0070 0x7f5bc80a8050 secure :-1 s=READY pgs=20 cs=0 l=1 rev1=1 crypto rx=0x7f5bc400d660 tx=0x7f5bc400db20 comp rx=0 tx=0).stop 2024-09-14T03:38:44.724 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.722+0000 7f5bd4910640 1 -- 172.21.15.83:0/4089121259 shutdown_connections 2024-09-14T03:38:44.724 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.722+0000 7f5bd4910640 1 --2- 172.21.15.83:0/4089121259 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f5bc80b0070 0x7f5bc80a8050 unknown :-1 s=CLOSED pgs=20 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:44.724 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.722+0000 7f5bd4910640 1 --2- 172.21.15.83:0/4089121259 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f5bc80af750 0x7f5bc80a7b10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:44.724 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.722+0000 7f5bd4910640 1 --2- 172.21.15.83:0/4089121259 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f5bc80182f0 0x7f5bc80a75d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:44.724 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.722+0000 7f5bd4910640 1 -- 172.21.15.83:0/4089121259 >> 172.21.15.83:0/4089121259 conn(0x7f5bc8005c20 msgr2=0x7f5bc8016370 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:38:44.724 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.722+0000 7f5bd4910640 1 -- 172.21.15.83:0/4089121259 shutdown_connections 2024-09-14T03:38:44.724 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.722+0000 7f5bd4910640 1 -- 172.21.15.83:0/4089121259 wait complete. 2024-09-14T03:38:44.725 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.724+0000 7f1f2ffff640 1 -- 172.21.15.83:0/1198565397 <== mon.0 v2:172.21.15.83:3300/0 8 ==== mon_command_ack([{prefix=config set, name=osd_op_complaint_time}]=0 v19)=0 v19) ==== 121+0+0 (secure 0 0 0) 0x7f1f24004b90 con 0x7f1f38095800 2024-09-14T03:38:44.726 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.725+0000 7f1f3db9c640 1 -- 172.21.15.83:0/1198565397 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f1f38095800 msgr2=0x7f1f3806f580 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:38:44.726 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.725+0000 7f1f3db9c640 1 --2- 172.21.15.83:0/1198565397 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f1f38095800 0x7f1f3806f580 secure :-1 s=READY pgs=26 cs=0 l=1 rev1=1 crypto rx=0x7f1f24000c00 tx=0x7f1f24003bf0 comp rx=0 tx=0).stop 2024-09-14T03:38:44.726 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.725+0000 7f1f3db9c640 1 -- 172.21.15.83:0/1198565397 shutdown_connections 2024-09-14T03:38:44.726 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.725+0000 7f1f3db9c640 1 --2- 172.21.15.83:0/1198565397 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f1f380727f0 0x7f1f38072c70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:44.727 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.725+0000 7f1f3db9c640 1 --2- 172.21.15.83:0/1198565397 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f1f38095800 0x7f1f3806f580 unknown :-1 s=CLOSED pgs=26 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:44.728 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.725+0000 7f1f3db9c640 1 --2- 172.21.15.83:0/1198565397 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f1f38094aa0 0x7f1f3806f040 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:44.728 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.725+0000 7f1f3db9c640 1 -- 172.21.15.83:0/1198565397 >> 172.21.15.83:0/1198565397 conn(0x7f1f3809c680 msgr2=0x7f1f38091420 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:38:44.728 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.725+0000 7f1f3db9c640 1 -- 172.21.15.83:0/1198565397 shutdown_connections 2024-09-14T03:38:44.728 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.726+0000 7f1f3db9c640 1 -- 172.21.15.83:0/1198565397 wait complete. 2024-09-14T03:38:44.735 DEBUG:tasks.ceph:waiting for RemoteProcess(client=, args=['sudo', 'ceph', '--cluster', 'ceph', 'config', 'set', 'mds', 'rados_mon_op_timeout', '900'], check_status=True, hostname='smithi083') 2024-09-14T03:38:44.735 DEBUG:tasks.ceph:waiting for RemoteProcess(client=, args=['sudo', 'ceph', '--cluster', 'ceph', 'config', 'set', 'mds', 'rados_osd_op_timeout', '900'], check_status=True, hostname='smithi083') 2024-09-14T03:38:44.735 DEBUG:tasks.ceph:waiting for RemoteProcess(client=, args=['sudo', 'ceph', '--cluster', 'ceph', 'config', 'set', 'mon', 'mon_op_complaint_time', '120'], check_status=True, hostname='smithi083') 2024-09-14T03:38:44.740 DEBUG:tasks.ceph:waiting for RemoteProcess(client=, args=['sudo', 'ceph', '--cluster', 'ceph', 'config', 'set', 'mon', 'mon_warn_on_pool_no_app', 'False'], check_status=True, hostname='smithi083') 2024-09-14T03:38:44.740 DEBUG:tasks.ceph:waiting for RemoteProcess(client=, args=['sudo', 'ceph', '--cluster', 'ceph', 'config', 'set', 'osd', 'osd_op_complaint_time', '180'], check_status=True, hostname='smithi083') 2024-09-14T03:38:44.740 INFO:tasks.ceph.mgr.x.smithi083.stderr:2024-09-14T03:38:44.739+0000 7f5470a36040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-14T03:38:44.743 INFO:tasks.ceph:Setting crush tunables to default 2024-09-14T03:38:44.743 DEBUG:teuthology.orchestra.run.smithi083:> sudo ceph --cluster ceph osd crush tunables default 2024-09-14T03:38:44.781 INFO:tasks.ceph.mgr.y.smithi083.stderr:2024-09-14T03:38:44.779+0000 7f0c09b73040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-14T03:38:44.855 INFO:tasks.ceph.mgr.x.smithi083.stderr:2024-09-14T03:38:44.854+0000 7f5470a36040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-14T03:38:44.878 INFO:tasks.ceph.mgr.y.smithi083.stderr:2024-09-14T03:38:44.876+0000 7f0c09b73040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-14T03:38:44.882 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.880+0000 7fab45303640 1 -- 172.21.15.83:0/405346387 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fab40075330 msgr2=0x7fab40073090 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:38:44.882 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.880+0000 7fab45303640 1 --2- 172.21.15.83:0/405346387 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fab40075330 0x7fab40073090 secure :-1 s=READY pgs=28 cs=0 l=1 rev1=1 crypto rx=0x7fab28005fd0 tx=0x7fab2802a540 comp rx=0 tx=0).stop 2024-09-14T03:38:44.882 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.880+0000 7fab45303640 1 -- 172.21.15.83:0/405346387 shutdown_connections 2024-09-14T03:38:44.882 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.880+0000 7fab45303640 1 --2- 172.21.15.83:0/405346387 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fab4006e8c0 0x7fab4006ed40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:44.882 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.880+0000 7fab45303640 1 --2- 172.21.15.83:0/405346387 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fab40075330 0x7fab40073090 unknown :-1 s=CLOSED pgs=28 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:44.882 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.880+0000 7fab45303640 1 -- 172.21.15.83:0/405346387 >> 172.21.15.83:0/405346387 conn(0x7fab4009cbb0 msgr2=0x7fab4009cfc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:38:44.882 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.880+0000 7fab45303640 1 -- 172.21.15.83:0/405346387 shutdown_connections 2024-09-14T03:38:44.882 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.880+0000 7fab45303640 1 -- 172.21.15.83:0/405346387 wait complete. 2024-09-14T03:38:44.884 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.881+0000 7fab45303640 1 Processor -- start 2024-09-14T03:38:44.884 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.881+0000 7fab45303640 1 -- start start 2024-09-14T03:38:44.884 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.881+0000 7fab45303640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fab4006e8c0 0x7fab400924c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:38:44.885 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.881+0000 7fab45303640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fab40072cb0 0x7fab40092a00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:38:44.885 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.881+0000 7fab45303640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fab40075330 0x7fab40092f40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:38:44.885 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.881+0000 7fab45303640 1 -- --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7fab401367c0 con 0x7fab40072cb0 2024-09-14T03:38:44.885 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.881+0000 7fab45303640 1 -- --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7fab40136640 con 0x7fab40075330 2024-09-14T03:38:44.885 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.881+0000 7fab45303640 1 -- --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7fab40136940 con 0x7fab4006e8c0 2024-09-14T03:38:44.885 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.881+0000 7fab37fff640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fab40072cb0 0x7fab40092a00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:38:44.885 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.881+0000 7fab37fff640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fab40072cb0 0x7fab40092a00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3300/0 says I am v2:172.21.15.83:44012/0 (socket says 172.21.15.83:44012) 2024-09-14T03:38:44.885 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.881+0000 7fab37fff640 1 -- 172.21.15.83:0/3436812647 learned_addr learned my addr 172.21.15.83:0/3436812647 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:38:44.885 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.881+0000 7fab3effd640 1 --2- 172.21.15.83:0/3436812647 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fab4006e8c0 0x7fab400924c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:38:44.885 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.881+0000 7fab3f7fe640 1 --2- 172.21.15.83:0/3436812647 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fab40075330 0x7fab40092f40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:38:44.885 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.882+0000 7fab37fff640 1 -- 172.21.15.83:0/3436812647 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fab4006e8c0 msgr2=0x7fab400924c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:38:44.885 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.882+0000 7fab37fff640 1 --2- 172.21.15.83:0/3436812647 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fab4006e8c0 0x7fab400924c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:44.885 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.882+0000 7fab37fff640 1 -- 172.21.15.83:0/3436812647 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fab40075330 msgr2=0x7fab40092f40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:38:44.885 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.882+0000 7fab37fff640 1 --2- 172.21.15.83:0/3436812647 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fab40075330 0x7fab40092f40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:44.886 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.882+0000 7fab37fff640 1 -- 172.21.15.83:0/3436812647 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fab40095910 con 0x7fab40072cb0 2024-09-14T03:38:44.886 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.882+0000 7fab37fff640 1 --2- 172.21.15.83:0/3436812647 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fab40072cb0 0x7fab40092a00 secure :-1 s=READY pgs=29 cs=0 l=1 rev1=1 crypto rx=0x7fab28009850 tx=0x7fab280038a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:38:44.886 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.882+0000 7fab377fe640 1 -- 172.21.15.83:0/3436812647 <== mon.0 v2:172.21.15.83:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fab2803f070 con 0x7fab40072cb0 2024-09-14T03:38:44.886 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.882+0000 7fab377fe640 1 -- 172.21.15.83:0/3436812647 <== mon.0 v2:172.21.15.83:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fab280047a0 con 0x7fab40072cb0 2024-09-14T03:38:44.886 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.882+0000 7fab45303640 1 -- 172.21.15.83:0/3436812647 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fab40093480 con 0x7fab40072cb0 2024-09-14T03:38:44.886 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.882+0000 7fab377fe640 1 -- 172.21.15.83:0/3436812647 <== mon.0 v2:172.21.15.83:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fab28035030 con 0x7fab40072cb0 2024-09-14T03:38:44.886 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.882+0000 7fab45303640 1 -- 172.21.15.83:0/3436812647 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fab40094f40 con 0x7fab40072cb0 2024-09-14T03:38:44.886 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.882+0000 7fab377fe640 1 -- 172.21.15.83:0/3436812647 <== mon.0 v2:172.21.15.83:3300/0 4 ==== mgrmap(e 1) ==== 807+0+0 (secure 0 0 0) 0x7fab28042070 con 0x7fab40072cb0 2024-09-14T03:38:44.886 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.882+0000 7fab377fe640 1 -- 172.21.15.83:0/3436812647 <== mon.0 v2:172.21.15.83:3300/0 5 ==== osd_map(1..1 src has 1..1) ==== 725+0+0 (secure 0 0 0) 0x7fab2803b880 con 0x7fab40072cb0 2024-09-14T03:38:44.887 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.883+0000 7fab45303640 1 -- 172.21.15.83:0/3436812647 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fab4006ed40 con 0x7fab40072cb0 2024-09-14T03:38:44.887 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.885+0000 7fab377fe640 1 -- 172.21.15.83:0/3436812647 <== mon.0 v2:172.21.15.83:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+74432 (secure 0 0 0) 0x7fab2803bad0 con 0x7fab40072cb0 2024-09-14T03:38:44.950 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.949+0000 7fab45303640 1 -- 172.21.15.83:0/3436812647 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_command({"prefix": "osd crush tunables", "profile": "default"} v 0) -- 0x7fab40075950 con 0x7fab40072cb0 2024-09-14T03:38:44.954 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.952+0000 7fab377fe640 1 -- 172.21.15.83:0/3436812647 <== mon.0 v2:172.21.15.83:3300/0 7 ==== mon_command_ack([{"prefix": "osd crush tunables", "profile": "default"}]=0 adjusted tunables profile to default v2) ==== 124+0+0 (secure 0 0 0) 0x7fab2803bcb0 con 0x7fab40072cb0 2024-09-14T03:38:44.954 INFO:teuthology.orchestra.run.smithi083.stderr:adjusted tunables profile to default 2024-09-14T03:38:44.955 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.954+0000 7fab45303640 1 -- 172.21.15.83:0/3436812647 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fab40072cb0 msgr2=0x7fab40092a00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:38:44.956 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.954+0000 7fab45303640 1 --2- 172.21.15.83:0/3436812647 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fab40072cb0 0x7fab40092a00 secure :-1 s=READY pgs=29 cs=0 l=1 rev1=1 crypto rx=0x7fab28009850 tx=0x7fab280038a0 comp rx=0 tx=0).stop 2024-09-14T03:38:44.956 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.954+0000 7fab45303640 1 -- 172.21.15.83:0/3436812647 shutdown_connections 2024-09-14T03:38:44.956 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.954+0000 7fab45303640 1 --2- 172.21.15.83:0/3436812647 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fab40075330 0x7fab40092f40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:44.956 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.954+0000 7fab45303640 1 --2- 172.21.15.83:0/3436812647 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fab40072cb0 0x7fab40092a00 unknown :-1 s=CLOSED pgs=29 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:44.956 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.954+0000 7fab45303640 1 --2- 172.21.15.83:0/3436812647 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fab4006e8c0 0x7fab400924c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:44.956 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.954+0000 7fab45303640 1 -- 172.21.15.83:0/3436812647 >> 172.21.15.83:0/3436812647 conn(0x7fab4009cbb0 msgr2=0x7fab40097740 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:38:44.956 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.954+0000 7fab45303640 1 -- 172.21.15.83:0/3436812647 shutdown_connections 2024-09-14T03:38:44.956 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:44.954+0000 7fab45303640 1 -- 172.21.15.83:0/3436812647 wait complete. 2024-09-14T03:38:44.970 INFO:tasks.ceph:check_enable_crimson: False 2024-09-14T03:38:44.970 INFO:tasks.ceph:Starting osd daemons in cluster ceph... 2024-09-14T03:38:44.970 DEBUG:teuthology.orchestra.run.smithi083:> set -ex 2024-09-14T03:38:44.970 DEBUG:teuthology.orchestra.run.smithi083:> sudo dd if=/var/lib/ceph/osd/ceph-0/fsid of=/dev/stdout 2024-09-14T03:38:44.995 DEBUG:teuthology.orchestra.run.smithi083:> set -ex 2024-09-14T03:38:44.995 DEBUG:teuthology.orchestra.run.smithi083:> sudo dd if=/var/lib/ceph/osd/ceph-1/fsid of=/dev/stdout 2024-09-14T03:38:45.060 DEBUG:teuthology.orchestra.run.smithi083:> set -ex 2024-09-14T03:38:45.060 DEBUG:teuthology.orchestra.run.smithi083:> sudo dd if=/var/lib/ceph/osd/ceph-2/fsid of=/dev/stdout 2024-09-14T03:38:45.124 DEBUG:teuthology.orchestra.run.smithi083:> set -ex 2024-09-14T03:38:45.125 DEBUG:teuthology.orchestra.run.smithi083:> sudo dd if=/var/lib/ceph/osd/ceph-3/fsid of=/dev/stdout 2024-09-14T03:38:45.188 DEBUG:teuthology.orchestra.run.smithi083:> sudo ceph --cluster ceph osd new ff477614-343d-4d97-9df5-94933f9b904d 0 2024-09-14T03:38:45.193 INFO:tasks.ceph.mgr.x.smithi083.stderr:2024-09-14T03:38:45.191+0000 7f5470a36040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-14T03:38:45.193 INFO:tasks.ceph.mgr.y.smithi083.stderr:2024-09-14T03:38:45.191+0000 7f0c09b73040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-14T03:38:45.247 INFO:tasks.ceph.mgr.x.smithi083.stderr:2024-09-14T03:38:45.246+0000 7f5470a36040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-14T03:38:45.248 INFO:tasks.ceph.mgr.y.smithi083.stderr:2024-09-14T03:38:45.246+0000 7f0c09b73040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-14T03:38:45.295 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.293+0000 7f85b2bc3640 1 -- 172.21.15.83:0/1061090874 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f85ac12fba0 msgr2=0x7f85ac131f90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:38:45.296 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.293+0000 7f85b2bc3640 1 --2- 172.21.15.83:0/1061090874 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f85ac12fba0 0x7f85ac131f90 secure :-1 s=READY pgs=18 cs=0 l=1 rev1=1 crypto rx=0x7f85a4005fd0 tx=0x7f85a402aa10 comp rx=0 tx=0).stop 2024-09-14T03:38:45.296 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.293+0000 7f85b2bc3640 1 -- 172.21.15.83:0/1061090874 shutdown_connections 2024-09-14T03:38:45.296 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.293+0000 7f85b2bc3640 1 --2- 172.21.15.83:0/1061090874 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f85ac12fba0 0x7f85ac131f90 unknown :-1 s=CLOSED pgs=18 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:45.296 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.293+0000 7f85b2bc3640 1 --2- 172.21.15.83:0/1061090874 >> v2:172.21.15.83:3300/0 conn(0x7f85ac12ac20 0x7f85ac12f660 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:45.296 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.293+0000 7f85b2bc3640 1 -- 172.21.15.83:0/1061090874 >> 172.21.15.83:0/1061090874 conn(0x7f85ac09d0e0 msgr2=0x7f85ac0995b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:38:45.296 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.293+0000 7f85b2bc3640 1 -- 172.21.15.83:0/1061090874 shutdown_connections 2024-09-14T03:38:45.296 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.293+0000 7f85b2bc3640 1 -- 172.21.15.83:0/1061090874 wait complete. 2024-09-14T03:38:45.296 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.294+0000 7f85b2bc3640 1 Processor -- start 2024-09-14T03:38:45.296 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.294+0000 7f85b2bc3640 1 -- start start 2024-09-14T03:38:45.298 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.295+0000 7f85b2bc3640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f85ac12a130 0x7f85ac122130 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:38:45.298 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.295+0000 7f85b2bc3640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f85ac12ac20 0x7f85ac122670 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:38:45.298 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.295+0000 7f85b2bc3640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f85ac12fba0 0x7f85ac122bb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:38:45.298 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.295+0000 7f85b2bc3640 1 -- --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7f85ac132ff0 con 0x7f85ac12ac20 2024-09-14T03:38:45.298 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.295+0000 7f85b2bc3640 1 -- --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7f85ac132e70 con 0x7f85ac12a130 2024-09-14T03:38:45.298 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.295+0000 7f85b2bc3640 1 -- --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7f85ac133170 con 0x7f85ac12fba0 2024-09-14T03:38:45.299 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.295+0000 7f85b095f640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f85ac12a130 0x7f85ac122130 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:38:45.299 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.295+0000 7f85a3fff640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f85ac12ac20 0x7f85ac122670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:38:45.299 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.295+0000 7f85b1160640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f85ac12fba0 0x7f85ac122bb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:38:45.299 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.295+0000 7f85b095f640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f85ac12a130 0x7f85ac122130 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3301/0 says I am v2:172.21.15.83:37608/0 (socket says 172.21.15.83:37608) 2024-09-14T03:38:45.299 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.295+0000 7f85b095f640 1 -- 172.21.15.83:0/463359888 learned_addr learned my addr 172.21.15.83:0/463359888 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:38:45.299 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.295+0000 7f85b095f640 1 -- 172.21.15.83:0/463359888 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f85ac12fba0 msgr2=0x7f85ac122bb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:38:45.299 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.295+0000 7f85b095f640 1 --2- 172.21.15.83:0/463359888 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f85ac12fba0 0x7f85ac122bb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:45.299 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.295+0000 7f85b095f640 1 -- 172.21.15.83:0/463359888 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f85ac12ac20 msgr2=0x7f85ac122670 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:38:45.299 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.295+0000 7f85b095f640 1 --2- 172.21.15.83:0/463359888 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f85ac12ac20 0x7f85ac122670 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:45.299 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.295+0000 7f85b095f640 1 -- 172.21.15.83:0/463359888 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f85ac125850 con 0x7f85ac12a130 2024-09-14T03:38:45.299 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.295+0000 7f85b1160640 1 --2- 172.21.15.83:0/463359888 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f85ac12fba0 0x7f85ac122bb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:38:45.299 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.295+0000 7f85a3fff640 1 --2- 172.21.15.83:0/463359888 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f85ac12ac20 0x7f85ac122670 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:38:45.299 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.296+0000 7f85b095f640 1 --2- 172.21.15.83:0/463359888 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f85ac12a130 0x7f85ac122130 secure :-1 s=READY pgs=19 cs=0 l=1 rev1=1 crypto rx=0x7f85a800c450 tx=0x7f85a800c910 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:38:45.299 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.296+0000 7f85a17fa640 1 -- 172.21.15.83:0/463359888 <== mon.1 v2:172.21.15.83:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f85a80052d0 con 0x7f85ac12a130 2024-09-14T03:38:45.300 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.296+0000 7f85a17fa640 1 -- 172.21.15.83:0/463359888 <== mon.1 v2:172.21.15.83:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f85a8005470 con 0x7f85ac12a130 2024-09-14T03:38:45.300 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.296+0000 7f85b2bc3640 1 -- 172.21.15.83:0/463359888 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f85ac124ee0 con 0x7f85ac12a130 2024-09-14T03:38:45.300 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.296+0000 7f85a17fa640 1 -- 172.21.15.83:0/463359888 <== mon.1 v2:172.21.15.83:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f85a8012870 con 0x7f85ac12a130 2024-09-14T03:38:45.300 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.296+0000 7f85b2bc3640 1 -- 172.21.15.83:0/463359888 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7f85ac124430 con 0x7f85ac12a130 2024-09-14T03:38:45.300 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.296+0000 7f85a17fa640 1 -- 172.21.15.83:0/463359888 <== mon.1 v2:172.21.15.83:3301/0 4 ==== mgrmap(e 1) ==== 807+0+0 (secure 0 0 0) 0x7f85a8012a10 con 0x7f85ac12a130 2024-09-14T03:38:45.300 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.296+0000 7f85a17fa640 1 -- 172.21.15.83:0/463359888 <== mon.1 v2:172.21.15.83:3301/0 5 ==== osd_map(2..2 src has 1..2) ==== 725+0+0 (secure 0 0 0) 0x7f85a801b8b0 con 0x7f85ac12a130 2024-09-14T03:38:45.300 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.297+0000 7f85b2bc3640 1 -- 172.21.15.83:0/463359888 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f85ac12a510 con 0x7f85ac12a130 2024-09-14T03:38:45.302 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.301+0000 7f85a17fa640 1 -- 172.21.15.83:0/463359888 <== mon.1 v2:172.21.15.83:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+74432 (secure 0 0 0) 0x7f85a801bb00 con 0x7f85ac12a130 2024-09-14T03:38:45.371 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.370+0000 7f85b2bc3640 1 -- 172.21.15.83:0/463359888 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_command({"prefix": "osd new", "uuid": "ff477614-343d-4d97-9df5-94933f9b904d", "id": 0} v 0) -- 0x7f85ac075950 con 0x7f85ac12a130 2024-09-14T03:38:45.376 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.374+0000 7f85a17fa640 1 -- 172.21.15.83:0/463359888 <== mon.1 v2:172.21.15.83:3301/0 7 ==== mon_command_ack([{"prefix": "osd new", "uuid": "ff477614-343d-4d97-9df5-94933f9b904d", "id": 0}]=0 v3) ==== 112+0+2 (secure 0 0 0) 0x7f85a801bce0 con 0x7f85ac12a130 2024-09-14T03:38:45.376 INFO:teuthology.orchestra.run.smithi083.stdout:0 2024-09-14T03:38:45.377 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.375+0000 7f85b2bc3640 1 -- 172.21.15.83:0/463359888 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f85ac12a130 msgr2=0x7f85ac122130 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:38:45.377 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.375+0000 7f85b2bc3640 1 --2- 172.21.15.83:0/463359888 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f85ac12a130 0x7f85ac122130 secure :-1 s=READY pgs=19 cs=0 l=1 rev1=1 crypto rx=0x7f85a800c450 tx=0x7f85a800c910 comp rx=0 tx=0).stop 2024-09-14T03:38:45.377 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.375+0000 7f85b2bc3640 1 -- 172.21.15.83:0/463359888 shutdown_connections 2024-09-14T03:38:45.377 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.375+0000 7f85b2bc3640 1 --2- 172.21.15.83:0/463359888 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f85ac12fba0 0x7f85ac122bb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:45.377 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.375+0000 7f85b2bc3640 1 --2- 172.21.15.83:0/463359888 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f85ac12ac20 0x7f85ac122670 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:45.377 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.375+0000 7f85b2bc3640 1 --2- 172.21.15.83:0/463359888 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f85ac12a130 0x7f85ac122130 unknown :-1 s=CLOSED pgs=19 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:45.377 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.375+0000 7f85b2bc3640 1 -- 172.21.15.83:0/463359888 >> 172.21.15.83:0/463359888 conn(0x7f85ac09d0e0 msgr2=0x7f85ac071e60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:38:45.377 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.376+0000 7f85b2bc3640 1 -- 172.21.15.83:0/463359888 shutdown_connections 2024-09-14T03:38:45.378 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.376+0000 7f85b2bc3640 1 -- 172.21.15.83:0/463359888 wait complete. 2024-09-14T03:38:45.387 DEBUG:teuthology.orchestra.run.smithi083:> sudo ceph --cluster ceph osd new 562ec9a0-3f91-4a29-896d-94340e9fa36f 1 2024-09-14T03:38:45.396 INFO:tasks.ceph.mgr.x.smithi083.stderr:2024-09-14T03:38:45.394+0000 7f5470a36040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-14T03:38:45.396 INFO:tasks.ceph.mgr.y.smithi083.stderr:2024-09-14T03:38:45.395+0000 7f0c09b73040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-14T03:38:45.488 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.486+0000 7fabe689c640 1 -- 172.21.15.83:0/3888374577 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fabe0131610 msgr2=0x7fabe0133a00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:38:45.488 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.486+0000 7fabe689c640 1 --2- 172.21.15.83:0/3888374577 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fabe0131610 0x7fabe0133a00 secure :-1 s=READY pgs=32 cs=0 l=1 rev1=1 crypto rx=0x7fabd8005fd0 tx=0x7fabd802a780 comp rx=0 tx=0).stop 2024-09-14T03:38:45.488 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.486+0000 7fabe689c640 1 -- 172.21.15.83:0/3888374577 shutdown_connections 2024-09-14T03:38:45.488 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.486+0000 7fabe689c640 1 --2- 172.21.15.83:0/3888374577 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fabe0131610 0x7fabe0133a00 unknown :-1 s=CLOSED pgs=32 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:45.488 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.486+0000 7fabe689c640 1 --2- 172.21.15.83:0/3888374577 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fabe006eca0 0x7fabe01310d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:45.489 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.486+0000 7fabe689c640 1 -- 172.21.15.83:0/3888374577 >> 172.21.15.83:0/3888374577 conn(0x7fabe009cbb0 msgr2=0x7fabe009cfc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:38:45.489 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.486+0000 7fabe689c640 1 -- 172.21.15.83:0/3888374577 shutdown_connections 2024-09-14T03:38:45.489 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.486+0000 7fabe689c640 1 -- 172.21.15.83:0/3888374577 wait complete. 2024-09-14T03:38:45.491 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.487+0000 7fabe689c640 1 Processor -- start 2024-09-14T03:38:45.491 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.487+0000 7fabe689c640 1 -- start start 2024-09-14T03:38:45.491 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.488+0000 7fabe689c640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fabe006e1b0 0x7fabe01c3d60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:38:45.491 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.488+0000 7fabe689c640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fabe006eca0 0x7fabe01c42a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:38:45.491 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.488+0000 7fabe689c640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fabe0131610 0x7fabe006f2c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:38:45.491 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.488+0000 7fabe689c640 1 -- --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7fabe01346e0 con 0x7fabe006eca0 2024-09-14T03:38:45.491 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.488+0000 7fabe689c640 1 -- --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7fabe0134560 con 0x7fabe0131610 2024-09-14T03:38:45.492 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.488+0000 7fabdffff640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fabe006e1b0 0x7fabe01c3d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:38:45.492 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.488+0000 7fabe689c640 1 -- --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7fabe0134860 con 0x7fabe006e1b0 2024-09-14T03:38:45.492 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.488+0000 7fabdffff640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fabe006e1b0 0x7fabe01c3d60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3302/0 says I am v2:172.21.15.83:60426/0 (socket says 172.21.15.83:60426) 2024-09-14T03:38:45.492 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.488+0000 7fabdffff640 1 -- 172.21.15.83:0/40362426 learned_addr learned my addr 172.21.15.83:0/40362426 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:38:45.492 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.488+0000 7fabdf7fe640 1 --2- 172.21.15.83:0/40362426 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fabe006eca0 0x7fabe01c42a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:38:45.492 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.488+0000 7fabe4e39640 1 --2- 172.21.15.83:0/40362426 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fabe0131610 0x7fabe006f2c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:38:45.492 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.488+0000 7fabdffff640 1 -- 172.21.15.83:0/40362426 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fabe0131610 msgr2=0x7fabe006f2c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:38:45.492 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.488+0000 7fabdffff640 1 --2- 172.21.15.83:0/40362426 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fabe0131610 0x7fabe006f2c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:45.492 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.488+0000 7fabdffff640 1 -- 172.21.15.83:0/40362426 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fabe006eca0 msgr2=0x7fabe01c42a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:38:45.492 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.488+0000 7fabdffff640 1 --2- 172.21.15.83:0/40362426 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fabe006eca0 0x7fabe01c42a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:45.492 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.488+0000 7fabdffff640 1 -- 172.21.15.83:0/40362426 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fabe012bdd0 con 0x7fabe006e1b0 2024-09-14T03:38:45.492 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.488+0000 7fabdf7fe640 1 --2- 172.21.15.83:0/40362426 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fabe006eca0 0x7fabe01c42a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:38:45.492 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.488+0000 7fabe4e39640 1 --2- 172.21.15.83:0/40362426 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fabe0131610 0x7fabe006f2c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:38:45.492 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.489+0000 7fabdffff640 1 --2- 172.21.15.83:0/40362426 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fabe006e1b0 0x7fabe01c3d60 secure :-1 s=READY pgs=21 cs=0 l=1 rev1=1 crypto rx=0x7fabd000c500 tx=0x7fabd000c9c0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:38:45.493 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.489+0000 7fabdcff9640 1 -- 172.21.15.83:0/40362426 <== mon.2 v2:172.21.15.83:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fabd0011790 con 0x7fabe006e1b0 2024-09-14T03:38:45.493 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.489+0000 7fabdcff9640 1 -- 172.21.15.83:0/40362426 <== mon.2 v2:172.21.15.83:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fabd0017070 con 0x7fabe006e1b0 2024-09-14T03:38:45.493 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.489+0000 7fabe689c640 1 -- 172.21.15.83:0/40362426 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fabe012aa20 con 0x7fabe006e1b0 2024-09-14T03:38:45.493 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.489+0000 7fabdcff9640 1 -- 172.21.15.83:0/40362426 <== mon.2 v2:172.21.15.83:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fabd0011cc0 con 0x7fabe006e1b0 2024-09-14T03:38:45.493 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.489+0000 7fabe689c640 1 -- 172.21.15.83:0/40362426 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7fabe01c47e0 con 0x7fabe006e1b0 2024-09-14T03:38:45.493 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.489+0000 7fabdcff9640 1 -- 172.21.15.83:0/40362426 <== mon.2 v2:172.21.15.83:3302/0 4 ==== mgrmap(e 1) ==== 807+0+0 (secure 0 0 0) 0x7fabd001c460 con 0x7fabe006e1b0 2024-09-14T03:38:45.493 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.489+0000 7fabdcff9640 1 -- 172.21.15.83:0/40362426 <== mon.2 v2:172.21.15.83:3302/0 5 ==== osd_map(3..3 src has 1..3) ==== 840+0+0 (secure 0 0 0) 0x7fabd0012900 con 0x7fabe006e1b0 2024-09-14T03:38:45.493 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.490+0000 7fabe689c640 1 -- 172.21.15.83:0/40362426 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fabe006e590 con 0x7fabe006e1b0 2024-09-14T03:38:45.494 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.492+0000 7fabdcff9640 1 -- 172.21.15.83:0/40362426 <== mon.2 v2:172.21.15.83:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+74432 (secure 0 0 0) 0x7fabd0012420 con 0x7fabe006e1b0 2024-09-14T03:38:45.563 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.561+0000 7fabe689c640 1 -- 172.21.15.83:0/40362426 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_command({"prefix": "osd new", "uuid": "562ec9a0-3f91-4a29-896d-94340e9fa36f", "id": 1} v 0) -- 0x7fabe0075950 con 0x7fabe006e1b0 2024-09-14T03:38:45.566 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.565+0000 7fabdcff9640 1 -- 172.21.15.83:0/40362426 <== mon.2 v2:172.21.15.83:3302/0 7 ==== mon_command_ack([{"prefix": "osd new", "uuid": "562ec9a0-3f91-4a29-896d-94340e9fa36f", "id": 1}]=0 v4) ==== 112+0+2 (secure 0 0 0) 0x7fabd0012bd0 con 0x7fabe006e1b0 2024-09-14T03:38:45.567 INFO:teuthology.orchestra.run.smithi083.stdout:1 2024-09-14T03:38:45.568 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.566+0000 7fabe689c640 1 -- 172.21.15.83:0/40362426 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fabe006e1b0 msgr2=0x7fabe01c3d60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:38:45.568 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.566+0000 7fabe689c640 1 --2- 172.21.15.83:0/40362426 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fabe006e1b0 0x7fabe01c3d60 secure :-1 s=READY pgs=21 cs=0 l=1 rev1=1 crypto rx=0x7fabd000c500 tx=0x7fabd000c9c0 comp rx=0 tx=0).stop 2024-09-14T03:38:45.568 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.566+0000 7fabe689c640 1 -- 172.21.15.83:0/40362426 shutdown_connections 2024-09-14T03:38:45.569 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.566+0000 7fabe689c640 1 --2- 172.21.15.83:0/40362426 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fabe0131610 0x7fabe006f2c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:45.569 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.566+0000 7fabe689c640 1 --2- 172.21.15.83:0/40362426 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fabe006eca0 0x7fabe01c42a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:45.569 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.566+0000 7fabe689c640 1 --2- 172.21.15.83:0/40362426 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fabe006e1b0 0x7fabe01c3d60 unknown :-1 s=CLOSED pgs=21 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:45.569 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.566+0000 7fabe689c640 1 -- 172.21.15.83:0/40362426 >> 172.21.15.83:0/40362426 conn(0x7fabe009cbb0 msgr2=0x7fabe0097e00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:38:45.569 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.566+0000 7fabe689c640 1 -- 172.21.15.83:0/40362426 shutdown_connections 2024-09-14T03:38:45.569 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.566+0000 7fabe689c640 1 -- 172.21.15.83:0/40362426 wait complete. 2024-09-14T03:38:45.578 DEBUG:teuthology.orchestra.run.smithi083:> sudo ceph --cluster ceph osd new 8c242dda-3308-4804-b2be-9a1737c42cf6 2 2024-09-14T03:38:45.723 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.719+0000 7fe1da648640 1 -- 172.21.15.83:0/2505631535 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fe1d4097260 msgr2=0x7fe1d4091610 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:38:45.724 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.719+0000 7fe1da648640 1 --2- 172.21.15.83:0/2505631535 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fe1d4097260 0x7fe1d4091610 secure :-1 s=READY pgs=20 cs=0 l=1 rev1=1 crypto rx=0x7fe1c4005fd0 tx=0x7fe1c402a540 comp rx=0 tx=0).stop 2024-09-14T03:38:45.724 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.719+0000 7fe1da648640 1 -- 172.21.15.83:0/2505631535 shutdown_connections 2024-09-14T03:38:45.724 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.719+0000 7fe1da648640 1 --2- 172.21.15.83:0/2505631535 >> v2:172.21.15.83:3300/0 conn(0x7fe1d4091b50 0x7fe1d4091fb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:45.724 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.719+0000 7fe1da648640 1 --2- 172.21.15.83:0/2505631535 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fe1d4097260 0x7fe1d4091610 unknown :-1 s=CLOSED pgs=20 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:45.724 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.719+0000 7fe1da648640 1 -- 172.21.15.83:0/2505631535 >> 172.21.15.83:0/2505631535 conn(0x7fe1d409d580 msgr2=0x7fe1d409d990 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:38:45.724 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.719+0000 7fe1da648640 1 -- 172.21.15.83:0/2505631535 shutdown_connections 2024-09-14T03:38:45.724 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.719+0000 7fe1da648640 1 -- 172.21.15.83:0/2505631535 wait complete. 2024-09-14T03:38:45.724 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.720+0000 7fe1da648640 1 Processor -- start 2024-09-14T03:38:45.724 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.720+0000 7fe1da648640 1 -- start start 2024-09-14T03:38:45.724 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.720+0000 7fe1da648640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fe1d4091b50 0x7fe1d41c3e30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:38:45.725 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.720+0000 7fe1da648640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fe1d4097260 0x7fe1d41c4370 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:38:45.725 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.720+0000 7fe1da648640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fe1d412f2d0 0x7fe1d412f6b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:38:45.725 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.720+0000 7fe1da648640 1 -- --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7fe1d41336d0 con 0x7fe1d412f2d0 2024-09-14T03:38:45.725 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.720+0000 7fe1da648640 1 -- --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7fe1d4133550 con 0x7fe1d4097260 2024-09-14T03:38:45.725 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.720+0000 7fe1da648640 1 -- --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7fe1d4133850 con 0x7fe1d4091b50 2024-09-14T03:38:45.725 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.721+0000 7fe1cbfff640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fe1d4097260 0x7fe1d41c4370 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:38:45.725 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.721+0000 7fe1d3fff640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fe1d4091b50 0x7fe1d41c3e30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:38:45.725 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.721+0000 7fe1cbfff640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fe1d4097260 0x7fe1d41c4370 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3301/0 says I am v2:172.21.15.83:37628/0 (socket says 172.21.15.83:37628) 2024-09-14T03:38:45.725 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.721+0000 7fe1cbfff640 1 -- 172.21.15.83:0/2231009402 learned_addr learned my addr 172.21.15.83:0/2231009402 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:38:45.725 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.721+0000 7fe1d8be5640 1 --2- 172.21.15.83:0/2231009402 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fe1d412f2d0 0x7fe1d412f6b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:38:45.725 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.721+0000 7fe1cbfff640 1 -- 172.21.15.83:0/2231009402 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fe1d4091b50 msgr2=0x7fe1d41c3e30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:38:45.725 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.721+0000 7fe1cbfff640 1 --2- 172.21.15.83:0/2231009402 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fe1d4091b50 0x7fe1d41c3e30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:45.725 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.721+0000 7fe1cbfff640 1 -- 172.21.15.83:0/2231009402 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fe1d412f2d0 msgr2=0x7fe1d412f6b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:38:45.726 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.721+0000 7fe1cbfff640 1 --2- 172.21.15.83:0/2231009402 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fe1d412f2d0 0x7fe1d412f6b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:45.726 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.721+0000 7fe1cbfff640 1 -- 172.21.15.83:0/2231009402 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fe1d4122420 con 0x7fe1d4097260 2024-09-14T03:38:45.726 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.721+0000 7fe1d3fff640 1 --2- 172.21.15.83:0/2231009402 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fe1d4091b50 0x7fe1d41c3e30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:38:45.726 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.721+0000 7fe1d8be5640 1 --2- 172.21.15.83:0/2231009402 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fe1d412f2d0 0x7fe1d412f6b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:38:45.726 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.721+0000 7fe1cbfff640 1 --2- 172.21.15.83:0/2231009402 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fe1d4097260 0x7fe1d41c4370 secure :-1 s=READY pgs=21 cs=0 l=1 rev1=1 crypto rx=0x7fe1bc00ea20 tx=0x7fe1bc00eee0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:38:45.726 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.721+0000 7fe1d17fa640 1 -- 172.21.15.83:0/2231009402 <== mon.1 v2:172.21.15.83:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe1bc00cd80 con 0x7fe1d4097260 2024-09-14T03:38:45.726 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.721+0000 7fe1d17fa640 1 -- 172.21.15.83:0/2231009402 <== mon.1 v2:172.21.15.83:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fe1bc010430 con 0x7fe1d4097260 2024-09-14T03:38:45.726 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.721+0000 7fe1d17fa640 1 -- 172.21.15.83:0/2231009402 <== mon.1 v2:172.21.15.83:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe1bc010840 con 0x7fe1d4097260 2024-09-14T03:38:45.726 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.721+0000 7fe1da648640 1 -- 172.21.15.83:0/2231009402 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fe1d41c48b0 con 0x7fe1d4097260 2024-09-14T03:38:45.726 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.721+0000 7fe1da648640 1 -- 172.21.15.83:0/2231009402 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7fe1d4122b70 con 0x7fe1d4097260 2024-09-14T03:38:45.726 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.722+0000 7fe1d17fa640 1 -- 172.21.15.83:0/2231009402 <== mon.1 v2:172.21.15.83:3301/0 4 ==== mgrmap(e 1) ==== 807+0+0 (secure 0 0 0) 0x7fe1bc0109e0 con 0x7fe1d4097260 2024-09-14T03:38:45.726 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.722+0000 7fe1d17fa640 1 -- 172.21.15.83:0/2231009402 <== mon.1 v2:172.21.15.83:3301/0 5 ==== osd_map(4..4 src has 1..4) ==== 955+0+0 (secure 0 0 0) 0x7fe1bc018dd0 con 0x7fe1d4097260 2024-09-14T03:38:45.727 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.723+0000 7fe1da648640 1 -- 172.21.15.83:0/2231009402 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fe1d4091fb0 con 0x7fe1d4097260 2024-09-14T03:38:45.727 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.725+0000 7fe1d17fa640 1 -- 172.21.15.83:0/2231009402 <== mon.1 v2:172.21.15.83:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+74432 (secure 0 0 0) 0x7fe1bc0213f0 con 0x7fe1d4097260 2024-09-14T03:38:45.728 INFO:tasks.ceph.mgr.x.smithi083.stderr:2024-09-14T03:38:45.727+0000 7f5470a36040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-14T03:38:45.729 INFO:tasks.ceph.mgr.y.smithi083.stderr:2024-09-14T03:38:45.727+0000 7f0c09b73040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-14T03:38:45.782 INFO:tasks.ceph.mgr.x.smithi083.stderr:2024-09-14T03:38:45.781+0000 7f5470a36040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-14T03:38:45.783 INFO:tasks.ceph.mgr.y.smithi083.stderr:2024-09-14T03:38:45.782+0000 7f0c09b73040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-14T03:38:45.809 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.808+0000 7fe1da648640 1 -- 172.21.15.83:0/2231009402 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_command({"prefix": "osd new", "uuid": "8c242dda-3308-4804-b2be-9a1737c42cf6", "id": 2} v 0) -- 0x7fe1d412f0b0 con 0x7fe1d4097260 2024-09-14T03:38:45.813 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.812+0000 7fe1d17fa640 1 -- 172.21.15.83:0/2231009402 <== mon.1 v2:172.21.15.83:3301/0 7 ==== mon_command_ack([{"prefix": "osd new", "uuid": "8c242dda-3308-4804-b2be-9a1737c42cf6", "id": 2}]=0 v5) ==== 112+0+2 (secure 0 0 0) 0x7fe1bc018420 con 0x7fe1d4097260 2024-09-14T03:38:45.813 INFO:teuthology.orchestra.run.smithi083.stdout:2 2024-09-14T03:38:45.815 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.813+0000 7fe1da648640 1 -- 172.21.15.83:0/2231009402 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fe1d4097260 msgr2=0x7fe1d41c4370 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:38:45.815 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.813+0000 7fe1da648640 1 --2- 172.21.15.83:0/2231009402 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fe1d4097260 0x7fe1d41c4370 secure :-1 s=READY pgs=21 cs=0 l=1 rev1=1 crypto rx=0x7fe1bc00ea20 tx=0x7fe1bc00eee0 comp rx=0 tx=0).stop 2024-09-14T03:38:45.815 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.813+0000 7fe1da648640 1 -- 172.21.15.83:0/2231009402 shutdown_connections 2024-09-14T03:38:45.815 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.813+0000 7fe1da648640 1 --2- 172.21.15.83:0/2231009402 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fe1d412f2d0 0x7fe1d412f6b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:45.815 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.813+0000 7fe1da648640 1 --2- 172.21.15.83:0/2231009402 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fe1d4097260 0x7fe1d41c4370 unknown :-1 s=CLOSED pgs=21 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:45.815 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.813+0000 7fe1da648640 1 --2- 172.21.15.83:0/2231009402 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fe1d4091b50 0x7fe1d41c3e30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:45.815 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.813+0000 7fe1da648640 1 -- 172.21.15.83:0/2231009402 >> 172.21.15.83:0/2231009402 conn(0x7fe1d409d580 msgr2=0x7fe1d40993b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:38:45.815 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.813+0000 7fe1da648640 1 -- 172.21.15.83:0/2231009402 shutdown_connections 2024-09-14T03:38:45.816 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.813+0000 7fe1da648640 1 -- 172.21.15.83:0/2231009402 wait complete. 2024-09-14T03:38:45.829 DEBUG:teuthology.orchestra.run.smithi083:> sudo ceph --cluster ceph osd new c4dd2c36-3c86-4b56-86fb-fede641a436c 3 2024-09-14T03:38:45.874 INFO:tasks.ceph.mgr.x.smithi083.stderr:2024-09-14T03:38:45.873+0000 7f5470a36040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-14T03:38:45.875 INFO:tasks.ceph.mgr.y.smithi083.stderr:2024-09-14T03:38:45.874+0000 7f0c09b73040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-14T03:38:45.929 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.927+0000 7fa6380a3640 1 -- 172.21.15.83:0/2991586967 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fa6300a0bf0 msgr2=0x7fa63009b520 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:38:45.929 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.927+0000 7fa6380a3640 1 --2- 172.21.15.83:0/2991586967 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fa6300a0bf0 0x7fa63009b520 secure :-1 s=READY pgs=22 cs=0 l=1 rev1=1 crypto rx=0x7fa620009860 tx=0x7fa62002a380 comp rx=0 tx=0).stop 2024-09-14T03:38:45.929 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.927+0000 7fa6380a3640 1 -- 172.21.15.83:0/2991586967 shutdown_connections 2024-09-14T03:38:45.930 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.927+0000 7fa6380a3640 1 --2- 172.21.15.83:0/2991586967 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fa630132470 0x7fa630134860 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:45.930 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.927+0000 7fa6380a3640 1 --2- 172.21.15.83:0/2991586967 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fa6300a0bf0 0x7fa63009b520 unknown :-1 s=CLOSED pgs=22 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:45.930 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.927+0000 7fa6380a3640 1 --2- 172.21.15.83:0/2991586967 >> v2:172.21.15.83:3300/0 conn(0x7fa63009ac00 0x7fa63009afe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:45.930 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.927+0000 7fa6380a3640 1 -- 172.21.15.83:0/2991586967 >> 172.21.15.83:0/2991586967 conn(0x7fa63009e830 msgr2=0x7fa6300a0300 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:38:45.930 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.928+0000 7fa6380a3640 1 -- 172.21.15.83:0/2991586967 shutdown_connections 2024-09-14T03:38:45.930 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.928+0000 7fa6380a3640 1 -- 172.21.15.83:0/2991586967 wait complete. 2024-09-14T03:38:45.932 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.928+0000 7fa6380a3640 1 Processor -- start 2024-09-14T03:38:45.932 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.928+0000 7fa6380a3640 1 -- start start 2024-09-14T03:38:45.932 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.929+0000 7fa6380a3640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fa63009ac00 0x7fa630131720 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:38:45.932 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.929+0000 7fa6380a3640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fa6300a0bf0 0x7fa630131c60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:38:45.933 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.929+0000 7fa6380a3640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fa630132470 0x7fa63008edf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:38:45.933 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.929+0000 7fa6380a3640 1 -- --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7fa630135960 con 0x7fa630132470 2024-09-14T03:38:45.933 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.929+0000 7fa6380a3640 1 -- --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7fa6301357e0 con 0x7fa63009ac00 2024-09-14T03:38:45.933 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.929+0000 7fa6380a3640 1 -- --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7fa630135ae0 con 0x7fa6300a0bf0 2024-09-14T03:38:45.933 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.929+0000 7fa63563e640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fa6300a0bf0 0x7fa630131c60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:38:45.933 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.929+0000 7fa635e3f640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fa63009ac00 0x7fa630131720 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:38:45.933 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.929+0000 7fa636640640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fa630132470 0x7fa63008edf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:38:45.933 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.929+0000 7fa63563e640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fa6300a0bf0 0x7fa630131c60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3302/0 says I am v2:172.21.15.83:60454/0 (socket says 172.21.15.83:60454) 2024-09-14T03:38:45.933 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.929+0000 7fa635e3f640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fa63009ac00 0x7fa630131720 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3301/0 says I am v2:172.21.15.83:37646/0 (socket says 172.21.15.83:37646) 2024-09-14T03:38:45.933 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.929+0000 7fa63563e640 1 -- 172.21.15.83:0/1113350054 learned_addr learned my addr 172.21.15.83:0/1113350054 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:38:45.933 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.929+0000 7fa636640640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fa630132470 0x7fa63008edf0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3300/0 says I am v2:172.21.15.83:44106/0 (socket says 172.21.15.83:44106) 2024-09-14T03:38:45.933 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.929+0000 7fa63563e640 1 -- 172.21.15.83:0/1113350054 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fa63009ac00 msgr2=0x7fa630131720 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:38:45.933 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.929+0000 7fa63563e640 1 --2- 172.21.15.83:0/1113350054 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fa63009ac00 0x7fa630131720 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:45.933 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.929+0000 7fa63563e640 1 -- 172.21.15.83:0/1113350054 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fa630132470 msgr2=0x7fa63008edf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:38:45.934 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.929+0000 7fa63563e640 1 --2- 172.21.15.83:0/1113350054 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fa630132470 0x7fa63008edf0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:45.934 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.929+0000 7fa63563e640 1 -- 172.21.15.83:0/1113350054 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fa630090920 con 0x7fa6300a0bf0 2024-09-14T03:38:45.934 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.929+0000 7fa636640640 1 --2- 172.21.15.83:0/1113350054 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fa630132470 0x7fa63008edf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:38:45.934 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.929+0000 7fa63563e640 1 --2- 172.21.15.83:0/1113350054 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fa6300a0bf0 0x7fa630131c60 secure :-1 s=READY pgs=23 cs=0 l=1 rev1=1 crypto rx=0x7fa620009780 tx=0x7fa620033b70 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:38:45.934 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.930+0000 7fa6267fc640 1 -- 172.21.15.83:0/1113350054 <== mon.2 v2:172.21.15.83:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa62003f020 con 0x7fa6300a0bf0 2024-09-14T03:38:45.934 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.930+0000 7fa6267fc640 1 -- 172.21.15.83:0/1113350054 <== mon.2 v2:172.21.15.83:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fa620033d60 con 0x7fa6300a0bf0 2024-09-14T03:38:45.934 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.930+0000 7fa6380a3640 1 -- 172.21.15.83:0/1113350054 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fa630091150 con 0x7fa6300a0bf0 2024-09-14T03:38:45.934 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.930+0000 7fa6267fc640 1 -- 172.21.15.83:0/1113350054 <== mon.2 v2:172.21.15.83:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa620037040 con 0x7fa6300a0bf0 2024-09-14T03:38:45.934 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.930+0000 7fa6380a3640 1 -- 172.21.15.83:0/1113350054 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7fa630091b90 con 0x7fa6300a0bf0 2024-09-14T03:38:45.934 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.930+0000 7fa6267fc640 1 -- 172.21.15.83:0/1113350054 <== mon.2 v2:172.21.15.83:3302/0 4 ==== mgrmap(e 1) ==== 807+0+0 (secure 0 0 0) 0x7fa620042020 con 0x7fa6300a0bf0 2024-09-14T03:38:45.934 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.930+0000 7fa6267fc640 1 -- 172.21.15.83:0/1113350054 <== mon.2 v2:172.21.15.83:3302/0 5 ==== osd_map(5..5 src has 1..5) ==== 1070+0+0 (secure 0 0 0) 0x7fa6200328d0 con 0x7fa6300a0bf0 2024-09-14T03:38:45.934 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.931+0000 7fa6380a3640 1 -- 172.21.15.83:0/1113350054 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fa630091940 con 0x7fa6300a0bf0 2024-09-14T03:38:45.935 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:45.933+0000 7fa6267fc640 1 -- 172.21.15.83:0/1113350054 <== mon.2 v2:172.21.15.83:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+74432 (secure 0 0 0) 0x7fa630091940 con 0x7fa6300a0bf0 2024-09-14T03:38:45.974 INFO:tasks.ceph.mgr.x.smithi083.stderr:2024-09-14T03:38:45.973+0000 7f5470a36040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-14T03:38:45.974 INFO:tasks.ceph.mgr.y.smithi083.stderr:2024-09-14T03:38:45.973+0000 7f0c09b73040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-14T03:38:46.005 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:46.004+0000 7fa6380a3640 1 -- 172.21.15.83:0/1113350054 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_command({"prefix": "osd new", "uuid": "c4dd2c36-3c86-4b56-86fb-fede641a436c", "id": 3} v 0) -- 0x7fa63009afe0 con 0x7fa6300a0bf0 2024-09-14T03:38:46.010 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:46.008+0000 7fa6267fc640 1 -- 172.21.15.83:0/1113350054 <== mon.2 v2:172.21.15.83:3302/0 7 ==== mon_command_ack([{"prefix": "osd new", "uuid": "c4dd2c36-3c86-4b56-86fb-fede641a436c", "id": 3}]=0 v6) ==== 112+0+2 (secure 0 0 0) 0x7fa620049070 con 0x7fa6300a0bf0 2024-09-14T03:38:46.010 INFO:teuthology.orchestra.run.smithi083.stdout:3 2024-09-14T03:38:46.011 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:46.009+0000 7fa6380a3640 1 -- 172.21.15.83:0/1113350054 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fa6300a0bf0 msgr2=0x7fa630131c60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:38:46.011 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:46.009+0000 7fa6380a3640 1 --2- 172.21.15.83:0/1113350054 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fa6300a0bf0 0x7fa630131c60 secure :-1 s=READY pgs=23 cs=0 l=1 rev1=1 crypto rx=0x7fa620009780 tx=0x7fa620033b70 comp rx=0 tx=0).stop 2024-09-14T03:38:46.011 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:46.009+0000 7fa6380a3640 1 -- 172.21.15.83:0/1113350054 shutdown_connections 2024-09-14T03:38:46.011 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:46.009+0000 7fa6380a3640 1 --2- 172.21.15.83:0/1113350054 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fa630132470 0x7fa63008edf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:46.011 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:46.009+0000 7fa6380a3640 1 --2- 172.21.15.83:0/1113350054 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fa6300a0bf0 0x7fa630131c60 unknown :-1 s=CLOSED pgs=23 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:46.011 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:46.009+0000 7fa6380a3640 1 --2- 172.21.15.83:0/1113350054 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fa63009ac00 0x7fa630131720 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:46.011 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:46.009+0000 7fa6380a3640 1 -- 172.21.15.83:0/1113350054 >> 172.21.15.83:0/1113350054 conn(0x7fa63009e830 msgr2=0x7fa630094290 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:38:46.012 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:46.010+0000 7fa6380a3640 1 -- 172.21.15.83:0/1113350054 shutdown_connections 2024-09-14T03:38:46.012 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:46.010+0000 7fa6380a3640 1 -- 172.21.15.83:0/1113350054 wait complete. 2024-09-14T03:38:46.021 INFO:tasks.ceph.osd.0:Restarting daemon 2024-09-14T03:38:46.021 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster ceph -i 0 2024-09-14T03:38:46.023 INFO:tasks.ceph.osd.0:Started 2024-09-14T03:38:46.024 INFO:tasks.ceph.osd.1:Restarting daemon 2024-09-14T03:38:46.024 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster ceph -i 1 2024-09-14T03:38:46.025 INFO:tasks.ceph.mgr.x.smithi083.stderr:2024-09-14T03:38:46.024+0000 7f5470a36040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-14T03:38:46.025 INFO:tasks.ceph.mgr.y.smithi083.stderr:2024-09-14T03:38:46.024+0000 7f0c09b73040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-14T03:38:46.026 INFO:tasks.ceph.osd.1:Started 2024-09-14T03:38:46.026 INFO:tasks.ceph.osd.2:Restarting daemon 2024-09-14T03:38:46.026 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster ceph -i 2 2024-09-14T03:38:46.028 INFO:tasks.ceph.osd.2:Started 2024-09-14T03:38:46.028 INFO:tasks.ceph.osd.3:Restarting daemon 2024-09-14T03:38:46.028 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster ceph -i 3 2024-09-14T03:38:46.031 INFO:tasks.ceph.osd.3:Started 2024-09-14T03:38:46.031 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-09-14T03:38:46.166 INFO:tasks.ceph.mgr.x.smithi083.stderr:2024-09-14T03:38:46.165+0000 7f5470a36040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-14T03:38:46.181 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:46.180+0000 7fa0f843d640 1 -- 172.21.15.83:0/179699467 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fa0f012dc30 msgr2=0x7fa0f012e090 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:38:46.181 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:46.180+0000 7fa0f843d640 1 --2- 172.21.15.83:0/179699467 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fa0f012dc30 0x7fa0f012e090 secure :-1 s=READY pgs=26 cs=0 l=1 rev1=1 crypto rx=0x7fa0e0009830 tx=0x7fa0e002a3f0 comp rx=0 tx=0).stop 2024-09-14T03:38:46.182 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:46.180+0000 7fa0f843d640 1 -- 172.21.15.83:0/179699467 shutdown_connections 2024-09-14T03:38:46.182 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:46.180+0000 7fa0f843d640 1 --2- 172.21.15.83:0/179699467 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fa0f012e5d0 0x7fa0f0134f80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:46.182 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:46.180+0000 7fa0f843d640 1 --2- 172.21.15.83:0/179699467 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fa0f012dc30 0x7fa0f012e090 unknown :-1 s=CLOSED pgs=26 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:46.182 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:46.180+0000 7fa0f843d640 1 --2- 172.21.15.83:0/179699467 >> v2:172.21.15.83:3300/0 conn(0x7fa0f006eb50 0x7fa0f006ef30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:46.182 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:46.180+0000 7fa0f843d640 1 -- 172.21.15.83:0/179699467 >> 172.21.15.83:0/179699467 conn(0x7fa0f00a0a40 msgr2=0x7fa0f009cf10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:38:46.182 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:46.180+0000 7fa0f843d640 1 -- 172.21.15.83:0/179699467 shutdown_connections 2024-09-14T03:38:46.183 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:46.180+0000 7fa0f843d640 1 -- 172.21.15.83:0/179699467 wait complete. 2024-09-14T03:38:46.185 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:46.181+0000 7fa0f843d640 1 Processor -- start 2024-09-14T03:38:46.185 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:46.181+0000 7fa0f843d640 1 -- start start 2024-09-14T03:38:46.185 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:46.181+0000 7fa0f843d640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fa0f006eb50 0x7fa0f0129f00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:38:46.185 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:46.181+0000 7fa0f843d640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fa0f012dc30 0x7fa0f012a440 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:38:46.185 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:46.181+0000 7fa0f843d640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fa0f012e5d0 0x7fa0f01c3fb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:38:46.185 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:46.182+0000 7fa0f59d8640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fa0f012dc30 0x7fa0f012a440 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:38:46.185 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:46.182+0000 7fa0f61d9640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fa0f006eb50 0x7fa0f0129f00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:38:46.185 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:46.182+0000 7fa0f843d640 1 -- --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7fa0f0135940 con 0x7fa0f006eb50 2024-09-14T03:38:46.185 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:46.182+0000 7fa0f843d640 1 -- --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7fa0f01357c0 con 0x7fa0f012dc30 2024-09-14T03:38:46.185 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:46.182+0000 7fa0f843d640 1 -- --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7fa0f0135ac0 con 0x7fa0f012e5d0 2024-09-14T03:38:46.186 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:46.182+0000 7fa0f69da640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fa0f012e5d0 0x7fa0f01c3fb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:38:46.186 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:46.182+0000 7fa0f61d9640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fa0f006eb50 0x7fa0f0129f00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3300/0 says I am v2:172.21.15.83:44152/0 (socket says 172.21.15.83:44152) 2024-09-14T03:38:46.186 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:46.182+0000 7fa0f61d9640 1 -- 172.21.15.83:0/1943913849 learned_addr learned my addr 172.21.15.83:0/1943913849 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:38:46.186 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:46.182+0000 7fa0f69da640 1 -- 172.21.15.83:0/1943913849 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fa0f012dc30 msgr2=0x7fa0f012a440 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:38:46.186 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:46.182+0000 7fa0f69da640 1 --2- 172.21.15.83:0/1943913849 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fa0f012dc30 0x7fa0f012a440 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:46.186 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:46.182+0000 7fa0f69da640 1 -- 172.21.15.83:0/1943913849 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fa0f006eb50 msgr2=0x7fa0f0129f00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:38:46.186 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:46.182+0000 7fa0f69da640 1 --2- 172.21.15.83:0/1943913849 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fa0f006eb50 0x7fa0f0129f00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:46.186 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:46.182+0000 7fa0f69da640 1 -- 172.21.15.83:0/1943913849 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fa0f012ee70 con 0x7fa0f012e5d0 2024-09-14T03:38:46.186 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:46.182+0000 7fa0f59d8640 1 --2- 172.21.15.83:0/1943913849 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fa0f012dc30 0x7fa0f012a440 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:38:46.186 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:46.182+0000 7fa0f61d9640 1 --2- 172.21.15.83:0/1943913849 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fa0f006eb50 0x7fa0f0129f00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:38:46.186 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:46.182+0000 7fa0f69da640 1 --2- 172.21.15.83:0/1943913849 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fa0f012e5d0 0x7fa0f01c3fb0 secure :-1 s=READY pgs=27 cs=0 l=1 rev1=1 crypto rx=0x7fa0e800ebc0 tx=0x7fa0e800c6a0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:38:46.186 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:46.182+0000 7fa0e6ffd640 1 -- 172.21.15.83:0/1943913849 <== mon.2 v2:172.21.15.83:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa0e8003b70 con 0x7fa0f012e5d0 2024-09-14T03:38:46.186 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:46.182+0000 7fa0e6ffd640 1 -- 172.21.15.83:0/1943913849 <== mon.2 v2:172.21.15.83:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fa0e8003d10 con 0x7fa0f012e5d0 2024-09-14T03:38:46.187 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:46.182+0000 7fa0f843d640 1 -- 172.21.15.83:0/1943913849 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fa0f012f340 con 0x7fa0f012e5d0 2024-09-14T03:38:46.187 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:46.182+0000 7fa0e6ffd640 1 -- 172.21.15.83:0/1943913849 <== mon.2 v2:172.21.15.83:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa0e8004280 con 0x7fa0f012e5d0 2024-09-14T03:38:46.187 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:46.182+0000 7fa0f843d640 1 -- 172.21.15.83:0/1943913849 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7fa0f012a980 con 0x7fa0f012e5d0 2024-09-14T03:38:46.187 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:46.183+0000 7fa0e6ffd640 1 -- 172.21.15.83:0/1943913849 <== mon.2 v2:172.21.15.83:3302/0 4 ==== mgrmap(e 1) ==== 807+0+0 (secure 0 0 0) 0x7fa0e8004420 con 0x7fa0f012e5d0 2024-09-14T03:38:46.187 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:46.183+0000 7fa0e6ffd640 1 -- 172.21.15.83:0/1943913849 <== mon.2 v2:172.21.15.83:3302/0 5 ==== osd_map(6..6 src has 1..6) ==== 1185+0+0 (secure 0 0 0) 0x7fa0e8004710 con 0x7fa0f012e5d0 2024-09-14T03:38:46.187 INFO:tasks.ceph.mgr.y.smithi083.stderr:2024-09-14T03:38:46.183+0000 7f0c09b73040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-14T03:38:46.188 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:46.184+0000 7fa0f843d640 1 -- 172.21.15.83:0/1943913849 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fa0f0098ec0 con 0x7fa0f012e5d0 2024-09-14T03:38:46.188 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:46.186+0000 7fa0e6ffd640 1 -- 172.21.15.83:0/1943913849 <== mon.2 v2:172.21.15.83:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+74432 (secure 0 0 0) 0x7fa0f0098ec0 con 0x7fa0f012e5d0 2024-09-14T03:38:46.222 INFO:tasks.ceph.mgr.x.smithi083.stderr:2024-09-14T03:38:46.220+0000 7f5470a36040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-14T03:38:46.238 INFO:tasks.ceph.mgr.y.smithi083.stderr:2024-09-14T03:38:46.236+0000 7f0c09b73040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-14T03:38:46.246 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:46.245+0000 7fa0f843d640 1 -- 172.21.15.83:0/1943913849 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) -- 0x7fa0f012e010 con 0x7fa0f012e5d0 2024-09-14T03:38:46.247 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:46.245+0000 7fa0e6ffd640 1 -- 172.21.15.83:0/1943913849 <== mon.2 v2:172.21.15.83:3302/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v6) ==== 74+0+4153 (secure 0 0 0) 0x7fa0f012f340 con 0x7fa0f012e5d0 2024-09-14T03:38:46.247 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:38:46.247 INFO:teuthology.orchestra.run.smithi083.stdout:{"epoch":6,"fsid":"765298ef-df4c-4f28-af02-b45f60264ff9","created":"2024-09-14T03:38:43.954952+0000","modified":"2024-09-14T03:38:46.006167+0000","last_up_change":"0.000000","last_in_change":"2024-09-14T03:38:46.006167+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"ff477614-343d-4d97-9df5-94933f9b904d","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"562ec9a0-3f91-4a29-896d-94340e9fa36f","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"8c242dda-3308-4804-b2be-9a1737c42cf6","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":3,"uuid":"c4dd2c36-3c86-4b56-86fb-fede641a436c","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-14T03:38:46.248 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:46.246+0000 7fa0f843d640 1 -- 172.21.15.83:0/1943913849 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fa0f012e5d0 msgr2=0x7fa0f01c3fb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:38:46.248 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:46.246+0000 7fa0f843d640 1 --2- 172.21.15.83:0/1943913849 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fa0f012e5d0 0x7fa0f01c3fb0 secure :-1 s=READY pgs=27 cs=0 l=1 rev1=1 crypto rx=0x7fa0e800ebc0 tx=0x7fa0e800c6a0 comp rx=0 tx=0).stop 2024-09-14T03:38:46.248 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:46.246+0000 7fa0f843d640 1 -- 172.21.15.83:0/1943913849 shutdown_connections 2024-09-14T03:38:46.248 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:46.246+0000 7fa0f843d640 1 --2- 172.21.15.83:0/1943913849 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fa0f012e5d0 0x7fa0f01c3fb0 unknown :-1 s=CLOSED pgs=27 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:46.248 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:46.246+0000 7fa0f843d640 1 --2- 172.21.15.83:0/1943913849 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fa0f012dc30 0x7fa0f012a440 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:46.249 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:46.246+0000 7fa0f843d640 1 --2- 172.21.15.83:0/1943913849 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fa0f006eb50 0x7fa0f0129f00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:46.249 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:46.246+0000 7fa0f843d640 1 -- 172.21.15.83:0/1943913849 >> 172.21.15.83:0/1943913849 conn(0x7fa0f00a0a40 msgr2=0x7fa0f01254d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:38:46.249 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:46.246+0000 7fa0f843d640 1 -- 172.21.15.83:0/1943913849 shutdown_connections 2024-09-14T03:38:46.249 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:38:46.247+0000 7fa0f843d640 1 -- 172.21.15.83:0/1943913849 wait complete. 2024-09-14T03:38:46.259 INFO:tasks.ceph.ceph_manager.ceph:[] 2024-09-14T03:38:46.259 INFO:tasks.ceph:Waiting for OSDs to come up 2024-09-14T03:38:46.269 INFO:tasks.ceph.mgr.x.smithi083.stderr:2024-09-14T03:38:46.268+0000 7f5470a36040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-14T03:38:46.281 INFO:tasks.ceph.mgr.y.smithi083.stderr:2024-09-14T03:38:46.280+0000 7f0c09b73040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-14T03:38:46.383 INFO:tasks.ceph.mgr.x.smithi083.stderr:2024-09-14T03:38:46.381+0000 7f5470a36040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-14T03:38:46.384 INFO:tasks.ceph.mgr.y.smithi083.stderr:2024-09-14T03:38:46.383+0000 7f0c09b73040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-14T03:38:46.434 INFO:tasks.ceph.mgr.x.smithi083.stderr:2024-09-14T03:38:46.433+0000 7f5470a36040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-14T03:38:46.435 INFO:tasks.ceph.mgr.y.smithi083.stderr:2024-09-14T03:38:46.434+0000 7f0c09b73040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-14T03:38:46.487 INFO:tasks.ceph.mgr.x.smithi083.stderr:2024-09-14T03:38:46.486+0000 7f5470a36040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-14T03:38:46.488 INFO:tasks.ceph.mgr.y.smithi083.stderr:2024-09-14T03:38:46.486+0000 7f0c09b73040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-14T03:38:46.558 INFO:tasks.ceph.mgr.x.smithi083.stderr:2024-09-14T03:38:46.556+0000 7f5470a36040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-14T03:38:46.558 INFO:tasks.ceph.mgr.y.smithi083.stderr:2024-09-14T03:38:46.556+0000 7f0c09b73040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-14T03:38:46.604 INFO:tasks.ceph.mgr.y.smithi083.stderr:2024-09-14T03:38:46.602+0000 7f0c09b73040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-14T03:38:46.604 INFO:tasks.ceph.mgr.x.smithi083.stderr:2024-09-14T03:38:46.602+0000 7f5470a36040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-14T03:38:46.649 INFO:tasks.ceph.mgr.x.smithi083.stderr:2024-09-14T03:38:46.647+0000 7f5470a36040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-14T03:38:46.649 INFO:tasks.ceph.mgr.y.smithi083.stderr:2024-09-14T03:38:46.647+0000 7f0c09b73040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-14T03:38:46.662 DEBUG:teuthology.orchestra.run.smithi083:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster ceph osd dump --format=json 2024-09-14T03:38:46.755 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:38:46.753+0000 7f122e824640 1 -- 172.21.15.83:0/737766379 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f122812b510 msgr2=0x7f122812fda0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:38:46.755 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:38:46.753+0000 7f122e824640 1 --2- 172.21.15.83:0/737766379 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f122812b510 0x7f122812fda0 secure :-1 s=READY pgs=40 cs=0 l=1 rev1=1 crypto rx=0x7f121c005fd0 tx=0x7f121c02a4b0 comp rx=0 tx=0).stop 2024-09-14T03:38:46.755 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:38:46.753+0000 7f122e824640 1 -- 172.21.15.83:0/737766379 shutdown_connections 2024-09-14T03:38:46.755 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:38:46.753+0000 7f122e824640 1 --2- 172.21.15.83:0/737766379 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f12281302e0 0x7f12281326d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:46.755 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:38:46.753+0000 7f122e824640 1 --2- 172.21.15.83:0/737766379 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f122812b510 0x7f122812fda0 unknown :-1 s=CLOSED pgs=40 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:46.755 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:38:46.753+0000 7f122e824640 1 -- 172.21.15.83:0/737766379 >> 172.21.15.83:0/737766379 conn(0x7f122809dad0 msgr2=0x7f122809dee0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:38:46.755 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:38:46.753+0000 7f122e824640 1 -- 172.21.15.83:0/737766379 shutdown_connections 2024-09-14T03:38:46.755 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:38:46.753+0000 7f122e824640 1 -- 172.21.15.83:0/737766379 wait complete. 2024-09-14T03:38:46.756 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:38:46.754+0000 7f122e824640 1 Processor -- start 2024-09-14T03:38:46.756 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:38:46.754+0000 7f122e824640 1 -- start start 2024-09-14T03:38:46.756 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:38:46.755+0000 7f122e824640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f122812aa20 0x7f12281c1e30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:38:46.756 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:38:46.755+0000 7f122e824640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f122812b510 0x7f12281c2370 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:38:46.756 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:38:46.755+0000 7f122e824640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f12281302e0 0x7f122812ca60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:38:46.757 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:38:46.755+0000 7f122e824640 1 -- --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7f12281337d0 con 0x7f122812aa20 2024-09-14T03:38:46.757 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:38:46.755+0000 7f122e824640 1 -- --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7f1228133650 con 0x7f122812b510 2024-09-14T03:38:46.757 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:38:46.755+0000 7f122e824640 1 -- --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7f1228133950 con 0x7f12281302e0 2024-09-14T03:38:46.757 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:38:46.755+0000 7f1227fff640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f122812aa20 0x7f12281c1e30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:38:46.757 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:38:46.755+0000 7f122cdc1640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f12281302e0 0x7f122812ca60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:38:46.757 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:38:46.755+0000 7f12277fe640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f122812b510 0x7f12281c2370 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:38:46.757 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:38:46.755+0000 7f1227fff640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f122812aa20 0x7f12281c1e30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3300/0 says I am v2:172.21.15.83:44172/0 (socket says 172.21.15.83:44172) 2024-09-14T03:38:46.757 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:38:46.755+0000 7f1227fff640 1 -- 172.21.15.83:0/574960922 learned_addr learned my addr 172.21.15.83:0/574960922 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:38:46.758 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:38:46.755+0000 7f12277fe640 1 -- 172.21.15.83:0/574960922 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f12281302e0 msgr2=0x7f122812ca60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:38:46.758 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:38:46.755+0000 7f12277fe640 1 --2- 172.21.15.83:0/574960922 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f12281302e0 0x7f122812ca60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:46.758 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:38:46.755+0000 7f12277fe640 1 -- 172.21.15.83:0/574960922 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f122812aa20 msgr2=0x7f12281c1e30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:38:46.758 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:38:46.755+0000 7f12277fe640 1 --2- 172.21.15.83:0/574960922 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f122812aa20 0x7f12281c1e30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:46.758 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:38:46.755+0000 7f12277fe640 1 -- 172.21.15.83:0/574960922 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f122812eb00 con 0x7f122812b510 2024-09-14T03:38:46.758 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:38:46.755+0000 7f1227fff640 1 --2- 172.21.15.83:0/574960922 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f122812aa20 0x7f12281c1e30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:38:46.759 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:38:46.755+0000 7f12277fe640 1 --2- 172.21.15.83:0/574960922 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f122812b510 0x7f12281c2370 secure :-1 s=READY pgs=23 cs=0 l=1 rev1=1 crypto rx=0x7f1228070e00 tx=0x7f121c005e80 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:38:46.759 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:38:46.756+0000 7f1224ff9640 1 -- 172.21.15.83:0/574960922 <== mon.1 v2:172.21.15.83:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f121c035030 con 0x7f122812b510 2024-09-14T03:38:46.759 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:38:46.756+0000 7f1224ff9640 1 -- 172.21.15.83:0/574960922 <== mon.1 v2:172.21.15.83:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f121c041070 con 0x7f122812b510 2024-09-14T03:38:46.759 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:38:46.756+0000 7f1224ff9640 1 -- 172.21.15.83:0/574960922 <== mon.1 v2:172.21.15.83:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f121c03d9c0 con 0x7f122812b510 2024-09-14T03:38:46.759 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:38:46.756+0000 7f122e824640 1 -- 172.21.15.83:0/574960922 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f1228113d00 con 0x7f122812b510 2024-09-14T03:38:46.759 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:38:46.756+0000 7f122e824640 1 -- 172.21.15.83:0/574960922 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7f12281c28b0 con 0x7f122812b510 2024-09-14T03:38:46.759 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:38:46.756+0000 7f1224ff9640 1 -- 172.21.15.83:0/574960922 <== mon.1 v2:172.21.15.83:3301/0 4 ==== mgrmap(e 1) ==== 807+0+0 (secure 0 0 0) 0x7f121c046470 con 0x7f122812b510 2024-09-14T03:38:46.759 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:38:46.756+0000 7f1224ff9640 1 -- 172.21.15.83:0/574960922 <== mon.1 v2:172.21.15.83:3301/0 5 ==== osd_map(6..6 src has 1..6) ==== 1185+0+0 (secure 0 0 0) 0x7f121c031910 con 0x7f122812b510 2024-09-14T03:38:46.759 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:38:46.757+0000 7f122e824640 1 -- 172.21.15.83:0/574960922 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f122812f3f0 con 0x7f122812b510 2024-09-14T03:38:46.760 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:38:46.759+0000 7f1224ff9640 1 -- 172.21.15.83:0/574960922 <== mon.1 v2:172.21.15.83:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+74432 (secure 0 0 0) 0x7f121c054020 con 0x7f122812b510 2024-09-14T03:38:46.818 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:38:46.817+0000 7f122e824640 1 -- 172.21.15.83:0/574960922 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) -- 0x7f122812ae00 con 0x7f122812b510 2024-09-14T03:38:46.819 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:38:46.818+0000 7f1224ff9640 1 -- 172.21.15.83:0/574960922 <== mon.1 v2:172.21.15.83:3301/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v6) ==== 74+0+4153 (secure 0 0 0) 0x7f121c031b10 con 0x7f122812b510 2024-09-14T03:38:46.819 INFO:teuthology.misc.health.smithi083.stdout: 2024-09-14T03:38:46.819 INFO:teuthology.misc.health.smithi083.stdout:{"epoch":6,"fsid":"765298ef-df4c-4f28-af02-b45f60264ff9","created":"2024-09-14T03:38:43.954952+0000","modified":"2024-09-14T03:38:46.006167+0000","last_up_change":"0.000000","last_in_change":"2024-09-14T03:38:46.006167+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"ff477614-343d-4d97-9df5-94933f9b904d","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"562ec9a0-3f91-4a29-896d-94340e9fa36f","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"8c242dda-3308-4804-b2be-9a1737c42cf6","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":3,"uuid":"c4dd2c36-3c86-4b56-86fb-fede641a436c","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-14T03:38:46.820 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:38:46.818+0000 7f122e824640 1 -- 172.21.15.83:0/574960922 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f122812b510 msgr2=0x7f12281c2370 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:38:46.820 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:38:46.819+0000 7f122e824640 1 --2- 172.21.15.83:0/574960922 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f122812b510 0x7f12281c2370 secure :-1 s=READY pgs=23 cs=0 l=1 rev1=1 crypto rx=0x7f1228070e00 tx=0x7f121c005e80 comp rx=0 tx=0).stop 2024-09-14T03:38:46.820 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:38:46.819+0000 7f122e824640 1 -- 172.21.15.83:0/574960922 shutdown_connections 2024-09-14T03:38:46.820 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:38:46.819+0000 7f122e824640 1 --2- 172.21.15.83:0/574960922 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f12281302e0 0x7f122812ca60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:46.820 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:38:46.819+0000 7f122e824640 1 --2- 172.21.15.83:0/574960922 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f122812b510 0x7f12281c2370 unknown :-1 s=CLOSED pgs=23 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:46.821 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:38:46.819+0000 7f122e824640 1 --2- 172.21.15.83:0/574960922 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f122812aa20 0x7f12281c1e30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:46.821 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:38:46.819+0000 7f122e824640 1 -- 172.21.15.83:0/574960922 >> 172.21.15.83:0/574960922 conn(0x7f122809dad0 msgr2=0x7f122806d860 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:38:46.821 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:38:46.819+0000 7f122e824640 1 -- 172.21.15.83:0/574960922 shutdown_connections 2024-09-14T03:38:46.821 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:38:46.819+0000 7f122e824640 1 -- 172.21.15.83:0/574960922 wait complete. 2024-09-14T03:38:46.829 DEBUG:teuthology.misc:0 of 4 OSDs are up 2024-09-14T03:38:46.887 INFO:tasks.ceph.mgr.x.smithi083.stderr:2024-09-14T03:38:46.886+0000 7f5470a36040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-14T03:38:46.888 INFO:tasks.ceph.mgr.y.smithi083.stderr:2024-09-14T03:38:46.886+0000 7f0c09b73040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-14T03:38:47.770 INFO:tasks.ceph.osd.1.smithi083.stderr:2024-09-14T03:38:47.768+0000 7fd5892a5640 -1 Falling back to public interface 2024-09-14T03:38:47.898 INFO:tasks.ceph.mgr.y.smithi083.stderr:2024-09-14T03:38:47.897+0000 7f0bf3372640 -1 mgr.server handle_report got status from non-daemon mon.b 2024-09-14T03:38:47.899 INFO:tasks.ceph.mgr.y.smithi083.stderr:2024-09-14T03:38:47.897+0000 7f0bf3372640 -1 mgr.server handle_report got status from non-daemon mon.c 2024-09-14T03:38:48.003 INFO:tasks.ceph.osd.0.smithi083.stderr:2024-09-14T03:38:48.001+0000 7fe48cc44640 -1 Falling back to public interface 2024-09-14T03:38:48.006 INFO:tasks.ceph.osd.3.smithi083.stderr:2024-09-14T03:38:48.005+0000 7f9a2a264640 -1 Falling back to public interface 2024-09-14T03:38:48.007 INFO:tasks.ceph.osd.2.smithi083.stderr:2024-09-14T03:38:48.005+0000 7f9a711db640 -1 Falling back to public interface 2024-09-14T03:38:48.007 INFO:tasks.ceph.osd.3.smithi083.stderr:2024-09-14T03:38:48.006+0000 7f9a2920d640 -1 PosixStack listen unable to listen on v1:0.0.0.0:6812/0: (98) Address already in use 2024-09-14T03:38:52.036 INFO:tasks.ceph.osd.0.smithi083.stderr:2024-09-14T03:38:52.034+0000 7fe48cc44640 -1 osd.0 0 log_to_monitors true 2024-09-14T03:38:52.357 INFO:tasks.ceph.osd.1.smithi083.stderr:2024-09-14T03:38:52.356+0000 7fd5892a5640 -1 osd.1 0 log_to_monitors true 2024-09-14T03:38:52.393 INFO:tasks.ceph.osd.2.smithi083.stderr:2024-09-14T03:38:52.392+0000 7f9a711db640 -1 osd.2 0 log_to_monitors true 2024-09-14T03:38:52.572 INFO:tasks.ceph.osd.3.smithi083.stderr:2024-09-14T03:38:52.570+0000 7f9a2a264640 -1 osd.3 0 log_to_monitors true 2024-09-14T03:38:53.233 DEBUG:teuthology.orchestra.run.smithi083:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster ceph osd dump --format=json 2024-09-14T03:38:53.329 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:38:53.327+0000 7f539b75e640 1 -- 172.21.15.83:0/3660740068 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f5394072b30 msgr2=0x7f539412b2d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:38:53.329 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:38:53.327+0000 7f539b75e640 1 --2- 172.21.15.83:0/3660740068 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f5394072b30 0x7f539412b2d0 secure :-1 s=READY pgs=47 cs=0 l=1 rev1=1 crypto rx=0x7f5384009860 tx=0x7f538402a380 comp rx=0 tx=0).stop 2024-09-14T03:38:53.329 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:38:53.327+0000 7f539b75e640 1 -- 172.21.15.83:0/3660740068 shutdown_connections 2024-09-14T03:38:53.330 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:38:53.327+0000 7f539b75e640 1 --2- 172.21.15.83:0/3660740068 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f539412ddc0 0x7f539412e210 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:53.330 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:38:53.327+0000 7f539b75e640 1 --2- 172.21.15.83:0/3660740068 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f5394072b30 0x7f539412b2d0 unknown :-1 s=CLOSED pgs=47 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:53.330 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:38:53.327+0000 7f539b75e640 1 --2- 172.21.15.83:0/3660740068 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f539412a880 0x7f539412ad90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:53.330 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:38:53.327+0000 7f539b75e640 1 -- 172.21.15.83:0/3660740068 >> 172.21.15.83:0/3660740068 conn(0x7f53940a0d80 msgr2=0x7f53940a1190 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:38:53.330 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:38:53.328+0000 7f539b75e640 1 -- 172.21.15.83:0/3660740068 shutdown_connections 2024-09-14T03:38:53.330 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:38:53.328+0000 7f539b75e640 1 -- 172.21.15.83:0/3660740068 wait complete. 2024-09-14T03:38:53.332 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:38:53.328+0000 7f539b75e640 1 Processor -- start 2024-09-14T03:38:53.332 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:38:53.328+0000 7f539b75e640 1 -- start start 2024-09-14T03:38:53.332 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:38:53.329+0000 7f539b75e640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f5394072b30 0x7f5394123a50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:38:53.332 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:38:53.329+0000 7f539b75e640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f539412a880 0x7f5394123f90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:38:53.332 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:38:53.329+0000 7f539b75e640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f539412ddc0 0x7f5394131b80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:38:53.332 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:38:53.329+0000 7f539b75e640 1 -- --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7f53941359c0 con 0x7f539412ddc0 2024-09-14T03:38:53.332 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:38:53.329+0000 7f539b75e640 1 -- --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7f5394135840 con 0x7f539412a880 2024-09-14T03:38:53.332 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:38:53.329+0000 7f539b75e640 1 -- --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7f5394135b40 con 0x7f5394072b30 2024-09-14T03:38:53.332 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:38:53.329+0000 7f53994fa640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f5394072b30 0x7f5394123a50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:38:53.332 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:38:53.329+0000 7f5399cfb640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f539412ddc0 0x7f5394131b80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:38:53.332 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:38:53.329+0000 7f53994fa640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f5394072b30 0x7f5394123a50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3302/0 says I am v2:172.21.15.83:45162/0 (socket says 172.21.15.83:45162) 2024-09-14T03:38:53.332 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:38:53.329+0000 7f5398cf9640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f539412a880 0x7f5394123f90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:38:53.333 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:38:53.329+0000 7f53994fa640 1 -- 172.21.15.83:0/567302036 learned_addr learned my addr 172.21.15.83:0/567302036 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:38:53.333 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:38:53.329+0000 7f53994fa640 1 -- 172.21.15.83:0/567302036 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f539412a880 msgr2=0x7f5394123f90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:38:53.333 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:38:53.329+0000 7f53994fa640 1 --2- 172.21.15.83:0/567302036 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f539412a880 0x7f5394123f90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:53.333 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:38:53.329+0000 7f53994fa640 1 -- 172.21.15.83:0/567302036 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f539412ddc0 msgr2=0x7f5394131b80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:38:53.333 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:38:53.329+0000 7f53994fa640 1 --2- 172.21.15.83:0/567302036 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f539412ddc0 0x7f5394131b80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:53.333 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:38:53.329+0000 7f53994fa640 1 -- 172.21.15.83:0/567302036 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f5394128570 con 0x7f5394072b30 2024-09-14T03:38:53.333 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:38:53.330+0000 7f53994fa640 1 --2- 172.21.15.83:0/567302036 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f5394072b30 0x7f5394123a50 secure :-1 s=READY pgs=35 cs=0 l=1 rev1=1 crypto rx=0x7f539000e9c0 tx=0x7f539000edc0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:38:53.333 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:38:53.330+0000 7f5389ffb640 1 -- 172.21.15.83:0/567302036 <== mon.2 v2:172.21.15.83:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f539000cd80 con 0x7f5394072b30 2024-09-14T03:38:53.333 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:38:53.330+0000 7f5389ffb640 1 -- 172.21.15.83:0/567302036 <== mon.2 v2:172.21.15.83:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f5390004020 con 0x7f5394072b30 2024-09-14T03:38:53.333 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:38:53.330+0000 7f539b75e640 1 -- 172.21.15.83:0/567302036 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f53941244d0 con 0x7f5394072b30 2024-09-14T03:38:53.333 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:38:53.330+0000 7f5389ffb640 1 -- 172.21.15.83:0/567302036 <== mon.2 v2:172.21.15.83:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5390004450 con 0x7f5394072b30 2024-09-14T03:38:53.333 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:38:53.330+0000 7f539b75e640 1 -- 172.21.15.83:0/567302036 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7f5394134030 con 0x7f5394072b30 2024-09-14T03:38:53.334 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:38:53.331+0000 7f5389ffb640 1 -- 172.21.15.83:0/567302036 <== mon.2 v2:172.21.15.83:3302/0 4 ==== mgrmap(e 4) ==== 55943+0+0 (secure 0 0 0) 0x7f53900035f0 con 0x7f5394072b30 2024-09-14T03:38:53.334 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:38:53.331+0000 7f5389ffb640 1 -- 172.21.15.83:0/567302036 <== mon.2 v2:172.21.15.83:3302/0 5 ==== osd_map(8..8 src has 1..8) ==== 1465+0+0 (secure 0 0 0) 0x7f539005dbc0 con 0x7f5394072b30 2024-09-14T03:38:53.334 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:38:53.331+0000 7f539b75e640 1 -- 172.21.15.83:0/567302036 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f539412e210 con 0x7f5394072b30 2024-09-14T03:38:53.337 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:38:53.336+0000 7f5389ffb640 1 -- 172.21.15.83:0/567302036 <== mon.2 v2:172.21.15.83:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+151260 (secure 0 0 0) 0x7f539003d470 con 0x7f5394072b30 2024-09-14T03:38:53.502 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:38:53.500+0000 7f539b75e640 1 -- 172.21.15.83:0/567302036 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) -- 0x7f5394134890 con 0x7f5394072b30 2024-09-14T03:38:53.502 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:38:53.501+0000 7f5389ffb640 1 -- 172.21.15.83:0/567302036 <== mon.2 v2:172.21.15.83:3302/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v8) ==== 74+0+4153 (secure 0 0 0) 0x7f539003d6d0 con 0x7f5394072b30 2024-09-14T03:38:53.502 INFO:teuthology.misc.health.smithi083.stdout: 2024-09-14T03:38:53.503 INFO:teuthology.misc.health.smithi083.stdout:{"epoch":8,"fsid":"765298ef-df4c-4f28-af02-b45f60264ff9","created":"2024-09-14T03:38:43.954952+0000","modified":"2024-09-14T03:38:53.100975+0000","last_up_change":"0.000000","last_in_change":"2024-09-14T03:38:46.006167+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"ff477614-343d-4d97-9df5-94933f9b904d","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"562ec9a0-3f91-4a29-896d-94340e9fa36f","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"8c242dda-3308-4804-b2be-9a1737c42cf6","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":3,"uuid":"c4dd2c36-3c86-4b56-86fb-fede641a436c","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-14T03:38:53.505 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:38:53.503+0000 7f539b75e640 1 -- 172.21.15.83:0/567302036 >> v1:172.21.15.83:6800/1934165865 conn(0x7f5378047f40 legacy=0x7f537804a400 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:38:53.505 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:38:53.503+0000 7f539b75e640 1 -- 172.21.15.83:0/567302036 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f5394072b30 msgr2=0x7f5394123a50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:38:53.505 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:38:53.503+0000 7f539b75e640 1 --2- 172.21.15.83:0/567302036 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f5394072b30 0x7f5394123a50 secure :-1 s=READY pgs=35 cs=0 l=1 rev1=1 crypto rx=0x7f539000e9c0 tx=0x7f539000edc0 comp rx=0 tx=0).stop 2024-09-14T03:38:53.505 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:38:53.503+0000 7f539b75e640 1 -- 172.21.15.83:0/567302036 shutdown_connections 2024-09-14T03:38:53.505 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:38:53.503+0000 7f539b75e640 1 --2- 172.21.15.83:0/567302036 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f539412ddc0 0x7f5394131b80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:53.506 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:38:53.503+0000 7f539b75e640 1 --2- 172.21.15.83:0/567302036 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f539412a880 0x7f5394123f90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:53.506 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:38:53.503+0000 7f539b75e640 1 --2- 172.21.15.83:0/567302036 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f5394072b30 0x7f5394123a50 unknown :-1 s=CLOSED pgs=35 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:38:53.506 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:38:53.503+0000 7f539b75e640 1 -- 172.21.15.83:0/567302036 >> 172.21.15.83:0/567302036 conn(0x7f53940a0d80 msgr2=0x7f539406f830 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:38:53.506 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:38:53.504+0000 7f539b75e640 1 -- 172.21.15.83:0/567302036 shutdown_connections 2024-09-14T03:38:53.506 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:38:53.504+0000 7f539b75e640 1 -- 172.21.15.83:0/567302036 wait complete. 2024-09-14T03:38:53.515 DEBUG:teuthology.misc:0 of 4 OSDs are up 2024-09-14T03:38:53.974 INFO:tasks.ceph.osd.0.smithi083.stderr:2024-09-14T03:38:53.973+0000 7fe4883b1640 -1 osd.0 0 waiting for initial osdmap 2024-09-14T03:38:53.977 INFO:tasks.ceph.osd.0.smithi083.stderr:2024-09-14T03:38:53.976+0000 7fe4809e3640 -1 osd.0 8 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-14T03:38:55.771 INFO:tasks.ceph.osd.3.smithi083.stderr:2024-09-14T03:38:55.769+0000 7f9a259d1640 -1 osd.3 0 waiting for initial osdmap 2024-09-14T03:38:55.776 INFO:tasks.ceph.osd.3.smithi083.stderr:2024-09-14T03:38:55.774+0000 7f9a1e003640 -1 osd.3 10 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-14T03:38:55.814 INFO:tasks.ceph.osd.1.smithi083.stderr:2024-09-14T03:38:55.812+0000 7fd584a12640 -1 osd.1 0 waiting for initial osdmap 2024-09-14T03:38:55.819 INFO:tasks.ceph.osd.1.smithi083.stderr:2024-09-14T03:38:55.817+0000 7fd57d044640 -1 osd.1 10 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-14T03:38:55.820 INFO:tasks.ceph.osd.2.smithi083.stderr:2024-09-14T03:38:55.817+0000 7f9a6c948640 -1 osd.2 0 waiting for initial osdmap 2024-09-14T03:38:55.827 INFO:tasks.ceph.osd.2.smithi083.stderr:2024-09-14T03:38:55.825+0000 7f9a64f7a640 -1 osd.2 10 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-14T03:38:59.918 DEBUG:teuthology.orchestra.run.smithi083:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster ceph osd dump --format=json 2024-09-14T03:39:00.011 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:39:00.009+0000 7fce67861640 1 -- 172.21.15.83:0/1290539260 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fce60131800 msgr2=0x7fce60133bf0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:00.011 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:39:00.009+0000 7fce67861640 1 --2- 172.21.15.83:0/1290539260 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fce60131800 0x7fce60133bf0 secure :-1 s=READY pgs=51 cs=0 l=1 rev1=1 crypto rx=0x7fce58005fd0 tx=0x7fce5802a780 comp rx=0 tx=0).stop 2024-09-14T03:39:00.011 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:39:00.009+0000 7fce67861640 1 -- 172.21.15.83:0/1290539260 shutdown_connections 2024-09-14T03:39:00.011 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:39:00.009+0000 7fce67861640 1 --2- 172.21.15.83:0/1290539260 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fce60131800 0x7fce60133bf0 unknown :-1 s=CLOSED pgs=51 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:00.011 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:39:00.009+0000 7fce67861640 1 --2- 172.21.15.83:0/1290539260 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fce6012c880 0x7fce601312c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:00.011 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:39:00.009+0000 7fce67861640 1 -- 172.21.15.83:0/1290539260 >> 172.21.15.83:0/1290539260 conn(0x7fce6009b780 msgr2=0x7fce6009bb90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:39:00.012 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:39:00.009+0000 7fce67861640 1 -- 172.21.15.83:0/1290539260 shutdown_connections 2024-09-14T03:39:00.012 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:39:00.009+0000 7fce67861640 1 -- 172.21.15.83:0/1290539260 wait complete. 2024-09-14T03:39:00.014 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:39:00.010+0000 7fce67861640 1 Processor -- start 2024-09-14T03:39:00.014 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:39:00.010+0000 7fce67861640 1 -- start start 2024-09-14T03:39:00.014 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:39:00.010+0000 7fce67861640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fce6012bd90 0x7fce601c4060 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:39:00.014 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:39:00.010+0000 7fce67861640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fce6012c880 0x7fce601c45a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:39:00.015 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:39:00.010+0000 7fce67861640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fce60131800 0x7fce6006e970 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:39:00.015 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:39:00.010+0000 7fce67861640 1 -- --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7fce60134980 con 0x7fce6012c880 2024-09-14T03:39:00.015 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:39:00.010+0000 7fce67861640 1 -- --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7fce60134800 con 0x7fce6012bd90 2024-09-14T03:39:00.015 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:39:00.010+0000 7fce67861640 1 -- --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7fce60134b00 con 0x7fce60131800 2024-09-14T03:39:00.015 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:39:00.010+0000 7fce64dfc640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fce6012c880 0x7fce601c45a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:39:00.015 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:39:00.010+0000 7fce655fd640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fce6012bd90 0x7fce601c4060 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:39:00.015 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:39:00.010+0000 7fce65dfe640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fce60131800 0x7fce6006e970 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:39:00.015 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:39:00.010+0000 7fce65dfe640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fce60131800 0x7fce6006e970 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3302/0 says I am v2:172.21.15.83:45200/0 (socket says 172.21.15.83:45200) 2024-09-14T03:39:00.015 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:39:00.010+0000 7fce65dfe640 1 -- 172.21.15.83:0/1721393990 learned_addr learned my addr 172.21.15.83:0/1721393990 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:39:00.015 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:39:00.010+0000 7fce64dfc640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fce6012c880 0x7fce601c45a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3300/0 says I am v2:172.21.15.83:39396/0 (socket says 172.21.15.83:39396) 2024-09-14T03:39:00.015 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:39:00.011+0000 7fce64dfc640 1 -- 172.21.15.83:0/1721393990 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fce60131800 msgr2=0x7fce6006e970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:00.015 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:39:00.011+0000 7fce64dfc640 1 --2- 172.21.15.83:0/1721393990 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fce60131800 0x7fce6006e970 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:00.016 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:39:00.011+0000 7fce64dfc640 1 -- 172.21.15.83:0/1721393990 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fce6012bd90 msgr2=0x7fce601c4060 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:00.016 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:39:00.011+0000 7fce64dfc640 1 --2- 172.21.15.83:0/1721393990 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fce6012bd90 0x7fce601c4060 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:00.016 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:39:00.011+0000 7fce64dfc640 1 -- 172.21.15.83:0/1721393990 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fce60070120 con 0x7fce6012c880 2024-09-14T03:39:00.016 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:39:00.011+0000 7fce65dfe640 1 --2- 172.21.15.83:0/1721393990 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fce60131800 0x7fce6006e970 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:39:00.016 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:39:00.011+0000 7fce64dfc640 1 --2- 172.21.15.83:0/1721393990 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fce6012c880 0x7fce601c45a0 secure :-1 s=READY pgs=52 cs=0 l=1 rev1=1 crypto rx=0x7fce5000e9f0 tx=0x7fce5000eeb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:39:00.016 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:39:00.011+0000 7fce55ffb640 1 -- 172.21.15.83:0/1721393990 <== mon.0 v2:172.21.15.83:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fce5000ccc0 con 0x7fce6012c880 2024-09-14T03:39:00.016 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:39:00.011+0000 7fce55ffb640 1 -- 172.21.15.83:0/1721393990 <== mon.0 v2:172.21.15.83:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fce50010430 con 0x7fce6012c880 2024-09-14T03:39:00.016 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:39:00.011+0000 7fce67861640 1 -- 172.21.15.83:0/1721393990 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fce600707e0 con 0x7fce6012c880 2024-09-14T03:39:00.016 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:39:00.011+0000 7fce55ffb640 1 -- 172.21.15.83:0/1721393990 <== mon.0 v2:172.21.15.83:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fce50010860 con 0x7fce6012c880 2024-09-14T03:39:00.016 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:39:00.011+0000 7fce67861640 1 -- 172.21.15.83:0/1721393990 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fce6008b650 con 0x7fce6012c880 2024-09-14T03:39:00.016 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:39:00.012+0000 7fce67861640 1 -- 172.21.15.83:0/1721393990 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fce6012c170 con 0x7fce6012c880 2024-09-14T03:39:00.016 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:39:00.012+0000 7fce55ffb640 1 -- 172.21.15.83:0/1721393990 <== mon.0 v2:172.21.15.83:3300/0 4 ==== mgrmap(e 4) ==== 55943+0+0 (secure 0 0 0) 0x7fce50010a00 con 0x7fce6012c880 2024-09-14T03:39:00.017 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:39:00.013+0000 7fce55ffb640 1 -- 172.21.15.83:0/1721393990 <== mon.0 v2:172.21.15.83:3300/0 5 ==== osd_map(14..14 src has 1..14) ==== 2459+0+0 (secure 0 0 0) 0x7fce5005e1c0 con 0x7fce6012c880 2024-09-14T03:39:00.018 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:39:00.016+0000 7fce55ffb640 1 -- 172.21.15.83:0/1721393990 <== mon.0 v2:172.21.15.83:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+151260 (secure 0 0 0) 0x7fce5003d660 con 0x7fce6012c880 2024-09-14T03:39:00.159 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:39:00.158+0000 7fce67861640 1 -- 172.21.15.83:0/1721393990 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) -- 0x7fce60098ea0 con 0x7fce6012c880 2024-09-14T03:39:00.160 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:39:00.158+0000 7fce55ffb640 1 -- 172.21.15.83:0/1721393990 <== mon.0 v2:172.21.15.83:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v14) ==== 74+0+7018 (secure 0 0 0) 0x7fce5003d8c0 con 0x7fce6012c880 2024-09-14T03:39:00.160 INFO:teuthology.misc.health.smithi083.stdout: 2024-09-14T03:39:00.160 INFO:teuthology.misc.health.smithi083.stdout:{"epoch":14,"fsid":"765298ef-df4c-4f28-af02-b45f60264ff9","created":"2024-09-14T03:38:43.954952+0000","modified":"2024-09-14T03:38:59.784286+0000","last_up_change":"2024-09-14T03:38:56.772236+0000","last_in_change":"2024-09-14T03:38:46.006167+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-14T03:38:56.922548+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"14","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":4,"score_stable":4,"optimal_score":0.5,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ff477614-343d-4d97-9df5-94933f9b904d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6805","nonce":2271402019}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6806","nonce":2271402019}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6808","nonce":2271402019}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6807","nonce":2271402019}]},"public_addr":"172.21.15.83:6805/2271402019","cluster_addr":"172.21.15.83:6806/2271402019","heartbeat_back_addr":"172.21.15.83:6808/2271402019","heartbeat_front_addr":"172.21.15.83:6807/2271402019","state":["exists","up"]},{"osd":1,"uuid":"562ec9a0-3f91-4a29-896d-94340e9fa36f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":11,"up_thru":12,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6801","nonce":2493296278}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6802","nonce":2493296278}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6804","nonce":2493296278}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6803","nonce":2493296278}]},"public_addr":"172.21.15.83:6801/2493296278","cluster_addr":"172.21.15.83:6802/2493296278","heartbeat_back_addr":"172.21.15.83:6804/2493296278","heartbeat_front_addr":"172.21.15.83:6803/2493296278","state":["exists","up"]},{"osd":2,"uuid":"8c242dda-3308-4804-b2be-9a1737c42cf6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":11,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6810","nonce":1553686396}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6812","nonce":1553686396}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6816","nonce":1553686396}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6814","nonce":1553686396}]},"public_addr":"172.21.15.83:6810/1553686396","cluster_addr":"172.21.15.83:6812/1553686396","heartbeat_back_addr":"172.21.15.83:6816/1553686396","heartbeat_front_addr":"172.21.15.83:6814/1553686396","state":["exists","up"]},{"osd":3,"uuid":"c4dd2c36-3c86-4b56-86fb-fede641a436c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":11,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6809","nonce":2350261690}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6811","nonce":2350261690}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6815","nonce":2350261690}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6813","nonce":2350261690}]},"public_addr":"172.21.15.83:6809/2350261690","cluster_addr":"172.21.15.83:6811/2350261690","heartbeat_back_addr":"172.21.15.83:6815/2350261690","heartbeat_front_addr":"172.21.15.83:6813/2350261690","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:38:53.047098+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:38:53.354351+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:38:53.422284+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:38:53.620657+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-14T03:39:00.162 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:39:00.160+0000 7fce67861640 1 -- 172.21.15.83:0/1721393990 >> v1:172.21.15.83:6800/1934165865 conn(0x7fce44047f70 legacy=0x7fce4404a430 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:00.162 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:39:00.160+0000 7fce67861640 1 -- 172.21.15.83:0/1721393990 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fce6012c880 msgr2=0x7fce601c45a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:00.163 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:39:00.160+0000 7fce67861640 1 --2- 172.21.15.83:0/1721393990 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fce6012c880 0x7fce601c45a0 secure :-1 s=READY pgs=52 cs=0 l=1 rev1=1 crypto rx=0x7fce5000e9f0 tx=0x7fce5000eeb0 comp rx=0 tx=0).stop 2024-09-14T03:39:00.163 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:39:00.160+0000 7fce67861640 1 -- 172.21.15.83:0/1721393990 shutdown_connections 2024-09-14T03:39:00.163 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:39:00.160+0000 7fce67861640 1 --2- 172.21.15.83:0/1721393990 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fce60131800 0x7fce6006e970 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:00.163 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:39:00.160+0000 7fce67861640 1 --2- 172.21.15.83:0/1721393990 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fce6012c880 0x7fce601c45a0 unknown :-1 s=CLOSED pgs=52 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:00.163 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:39:00.160+0000 7fce67861640 1 --2- 172.21.15.83:0/1721393990 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fce6012bd90 0x7fce601c4060 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:00.163 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:39:00.160+0000 7fce67861640 1 -- 172.21.15.83:0/1721393990 >> 172.21.15.83:0/1721393990 conn(0x7fce6009b780 msgr2=0x7fce60099af0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:39:00.163 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:39:00.160+0000 7fce67861640 1 -- 172.21.15.83:0/1721393990 shutdown_connections 2024-09-14T03:39:00.163 INFO:teuthology.misc.health.smithi083.stderr:2024-09-14T03:39:00.160+0000 7fce67861640 1 -- 172.21.15.83:0/1721393990 wait complete. 2024-09-14T03:39:00.171 DEBUG:teuthology.misc:4 of 4 OSDs are up 2024-09-14T03:39:00.171 INFO:tasks.ceph:Creating RBD pool 2024-09-14T03:39:00.172 DEBUG:teuthology.orchestra.run.smithi083:> sudo ceph --cluster ceph osd pool create rbd 8 2024-09-14T03:39:00.270 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:00.268+0000 7fd5bb0f7640 1 -- 172.21.15.83:0/894869135 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fd5b41303e0 msgr2=0x7fd5b41327d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:00.270 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:00.268+0000 7fd5bb0f7640 1 --2- 172.21.15.83:0/894869135 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fd5b41303e0 0x7fd5b41327d0 secure :-1 s=READY pgs=54 cs=0 l=1 rev1=1 crypto rx=0x7fd5ac009be0 tx=0x7fd5ac02a7c0 comp rx=0 tx=0).stop 2024-09-14T03:39:00.270 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:00.268+0000 7fd5bb0f7640 1 -- 172.21.15.83:0/894869135 shutdown_connections 2024-09-14T03:39:00.270 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:00.268+0000 7fd5bb0f7640 1 --2- 172.21.15.83:0/894869135 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fd5b41303e0 0x7fd5b41327d0 unknown :-1 s=CLOSED pgs=54 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:00.271 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:00.268+0000 7fd5bb0f7640 1 --2- 172.21.15.83:0/894869135 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fd5b4095a20 0x7fd5b4094e70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:00.271 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:00.268+0000 7fd5bb0f7640 1 -- 172.21.15.83:0/894869135 >> 172.21.15.83:0/894869135 conn(0x7fd5b409b580 msgr2=0x7fd5b409b990 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:39:00.271 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:00.268+0000 7fd5bb0f7640 1 -- 172.21.15.83:0/894869135 shutdown_connections 2024-09-14T03:39:00.271 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:00.268+0000 7fd5bb0f7640 1 -- 172.21.15.83:0/894869135 wait complete. 2024-09-14T03:39:00.273 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:00.269+0000 7fd5bb0f7640 1 Processor -- start 2024-09-14T03:39:00.273 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:00.269+0000 7fd5bb0f7640 1 -- start start 2024-09-14T03:39:00.273 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:00.270+0000 7fd5bb0f7640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fd5b4095a20 0x7fd5b4071a00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:39:00.273 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:00.270+0000 7fd5bb0f7640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fd5b40973b0 0x7fd5b4071f40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:39:00.273 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:00.270+0000 7fd5bb0f7640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fd5b41303e0 0x7fd5b4072480 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:39:00.273 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:00.270+0000 7fd5bb0f7640 1 -- --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7fd5b41338d0 con 0x7fd5b41303e0 2024-09-14T03:39:00.274 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:00.270+0000 7fd5bb0f7640 1 -- --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7fd5b4133750 con 0x7fd5b40973b0 2024-09-14T03:39:00.274 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:00.270+0000 7fd5bb0f7640 1 -- --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7fd5b4133a50 con 0x7fd5b4095a20 2024-09-14T03:39:00.274 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:00.270+0000 7fd5b3fff640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fd5b40973b0 0x7fd5b4071f40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:39:00.274 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:00.270+0000 7fd5b8e93640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fd5b4095a20 0x7fd5b4071a00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:39:00.274 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:00.270+0000 7fd5b9694640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fd5b41303e0 0x7fd5b4072480 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:39:00.274 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:00.270+0000 7fd5b3fff640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fd5b40973b0 0x7fd5b4071f40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3301/0 says I am v2:172.21.15.83:33940/0 (socket says 172.21.15.83:33940) 2024-09-14T03:39:00.274 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:00.270+0000 7fd5b3fff640 1 -- 172.21.15.83:0/3211757602 learned_addr learned my addr 172.21.15.83:0/3211757602 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:39:00.274 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:00.270+0000 7fd5b3fff640 1 -- 172.21.15.83:0/3211757602 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fd5b4095a20 msgr2=0x7fd5b4071a00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:00.274 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:00.270+0000 7fd5b3fff640 1 --2- 172.21.15.83:0/3211757602 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fd5b4095a20 0x7fd5b4071a00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:00.274 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:00.270+0000 7fd5b3fff640 1 -- 172.21.15.83:0/3211757602 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fd5b41303e0 msgr2=0x7fd5b4072480 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:00.274 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:00.270+0000 7fd5b3fff640 1 --2- 172.21.15.83:0/3211757602 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fd5b41303e0 0x7fd5b4072480 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:00.274 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:00.270+0000 7fd5b3fff640 1 -- 172.21.15.83:0/3211757602 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fd5b406be60 con 0x7fd5b40973b0 2024-09-14T03:39:00.274 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:00.270+0000 7fd5b8e93640 1 --2- 172.21.15.83:0/3211757602 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fd5b4095a20 0x7fd5b4071a00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:39:00.275 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:00.270+0000 7fd5b9694640 1 --2- 172.21.15.83:0/3211757602 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fd5b41303e0 0x7fd5b4072480 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:39:00.275 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:00.270+0000 7fd5b3fff640 1 --2- 172.21.15.83:0/3211757602 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fd5b40973b0 0x7fd5b4071f40 secure :-1 s=READY pgs=32 cs=0 l=1 rev1=1 crypto rx=0x7fd5a800e9c0 tx=0x7fd5a800ee80 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:39:00.275 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:00.270+0000 7fd5b17fa640 1 -- 172.21.15.83:0/3211757602 <== mon.1 v2:172.21.15.83:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd5a800cde0 con 0x7fd5b40973b0 2024-09-14T03:39:00.275 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:00.271+0000 7fd5b17fa640 1 -- 172.21.15.83:0/3211757602 <== mon.1 v2:172.21.15.83:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fd5a8004020 con 0x7fd5b40973b0 2024-09-14T03:39:00.275 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:00.271+0000 7fd5bb0f7640 1 -- 172.21.15.83:0/3211757602 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fd5b4072fe0 con 0x7fd5b40973b0 2024-09-14T03:39:00.275 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:00.271+0000 7fd5b17fa640 1 -- 172.21.15.83:0/3211757602 <== mon.1 v2:172.21.15.83:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd5a8004450 con 0x7fd5b40973b0 2024-09-14T03:39:00.275 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:00.271+0000 7fd5bb0f7640 1 -- 172.21.15.83:0/3211757602 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7fd5b4073700 con 0x7fd5b40973b0 2024-09-14T03:39:00.275 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:00.272+0000 7fd5bb0f7640 1 -- 172.21.15.83:0/3211757602 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fd5b4130ce0 con 0x7fd5b40973b0 2024-09-14T03:39:00.275 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:00.272+0000 7fd5b17fa640 1 -- 172.21.15.83:0/3211757602 <== mon.1 v2:172.21.15.83:3301/0 4 ==== mgrmap(e 4) ==== 55943+0+0 (secure 0 0 0) 0x7fd5a80035f0 con 0x7fd5b40973b0 2024-09-14T03:39:00.275 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:00.272+0000 7fd5b17fa640 1 -- 172.21.15.83:0/3211757602 <== mon.1 v2:172.21.15.83:3301/0 5 ==== osd_map(14..14 src has 1..14) ==== 2459+0+0 (secure 0 0 0) 0x7fd5a805e030 con 0x7fd5b40973b0 2024-09-14T03:39:00.277 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:00.276+0000 7fd5b17fa640 1 -- 172.21.15.83:0/3211757602 <== mon.1 v2:172.21.15.83:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+151260 (secure 0 0 0) 0x7fd5b4130ce0 con 0x7fd5b40973b0 2024-09-14T03:39:00.416 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:00.414+0000 7fd5bb0f7640 1 -- 172.21.15.83:0/3211757602 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_command({"prefix": "osd pool create", "pool": "rbd", "pg_num": 8} v 0) -- 0x7fd5b4094760 con 0x7fd5b40973b0 2024-09-14T03:39:00.799 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:00.797+0000 7fd5b17fa640 1 -- 172.21.15.83:0/3211757602 <== mon.1 v2:172.21.15.83:3301/0 7 ==== mon_command_ack([{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]=0 pool 'rbd' created v15) ==== 109+0+0 (secure 0 0 0) 0x7fd5a803d4d0 con 0x7fd5b40973b0 2024-09-14T03:39:00.799 INFO:teuthology.orchestra.run.smithi083.stderr:pool 'rbd' created 2024-09-14T03:39:00.802 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:00.800+0000 7fd5bb0f7640 1 -- 172.21.15.83:0/3211757602 >> v1:172.21.15.83:6800/1934165865 conn(0x7fd5a0047f40 legacy=0x7fd5a004a400 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:00.802 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:00.800+0000 7fd5bb0f7640 1 -- 172.21.15.83:0/3211757602 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fd5b40973b0 msgr2=0x7fd5b4071f40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:00.802 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:00.800+0000 7fd5bb0f7640 1 --2- 172.21.15.83:0/3211757602 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fd5b40973b0 0x7fd5b4071f40 secure :-1 s=READY pgs=32 cs=0 l=1 rev1=1 crypto rx=0x7fd5a800e9c0 tx=0x7fd5a800ee80 comp rx=0 tx=0).stop 2024-09-14T03:39:00.802 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:00.800+0000 7fd5bb0f7640 1 -- 172.21.15.83:0/3211757602 shutdown_connections 2024-09-14T03:39:00.802 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:00.800+0000 7fd5bb0f7640 1 --2- 172.21.15.83:0/3211757602 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fd5b41303e0 0x7fd5b4072480 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:00.802 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:00.800+0000 7fd5bb0f7640 1 --2- 172.21.15.83:0/3211757602 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fd5b40973b0 0x7fd5b4071f40 unknown :-1 s=CLOSED pgs=32 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:00.802 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:00.800+0000 7fd5bb0f7640 1 --2- 172.21.15.83:0/3211757602 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fd5b4095a20 0x7fd5b4071a00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:00.802 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:00.800+0000 7fd5bb0f7640 1 -- 172.21.15.83:0/3211757602 >> 172.21.15.83:0/3211757602 conn(0x7fd5b409b580 msgr2=0x7fd5b4098790 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:39:00.802 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:00.800+0000 7fd5bb0f7640 1 -- 172.21.15.83:0/3211757602 shutdown_connections 2024-09-14T03:39:00.802 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:00.800+0000 7fd5bb0f7640 1 -- 172.21.15.83:0/3211757602 wait complete. 2024-09-14T03:39:00.812 DEBUG:teuthology.orchestra.run.smithi083:> rbd --cluster ceph pool init rbd 2024-09-14T03:39:01.035 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:01.033+0000 7feb9c57dd80 1 -- 172.21.15.83:0/3979377490 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x5635ae64f0c0 msgr2=0x5635ae6514b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:01.035 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:01.033+0000 7feb9c57dd80 1 --2- 172.21.15.83:0/3979377490 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x5635ae64f0c0 0x5635ae6514b0 secure :-1 s=READY pgs=33 cs=0 l=1 rev1=1 crypto rx=0x7feb9400a600 tx=0x7feb94045ec0 comp rx=0 tx=0).stop 2024-09-14T03:39:01.036 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:01.034+0000 7feb9c57dd80 1 -- 172.21.15.83:0/3979377490 shutdown_connections 2024-09-14T03:39:01.036 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:01.034+0000 7feb9c57dd80 1 --2- 172.21.15.83:0/3979377490 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x5635ae64f0c0 0x5635ae6514b0 unknown :-1 s=CLOSED pgs=33 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:01.036 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:01.034+0000 7feb9c57dd80 1 --2- 172.21.15.83:0/3979377490 >> v2:172.21.15.83:3300/0 conn(0x5635ae646540 0x5635ae64eb80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:01.036 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:01.034+0000 7feb9c57dd80 1 -- 172.21.15.83:0/3979377490 >> 172.21.15.83:0/3979377490 conn(0x5635ae5b9060 msgr2=0x5635ae5b9470 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:39:01.036 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:01.034+0000 7feb9c57dd80 1 -- 172.21.15.83:0/3979377490 shutdown_connections 2024-09-14T03:39:01.036 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:01.034+0000 7feb9c57dd80 1 -- 172.21.15.83:0/3979377490 wait complete. 2024-09-14T03:39:01.038 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:01.034+0000 7feb9c57dd80 1 Processor -- start 2024-09-14T03:39:01.038 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:01.034+0000 7feb9c57dd80 1 -- start start 2024-09-14T03:39:01.038 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:01.035+0000 7feb9c57dd80 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x5635ae646e20 0x5635ae64a910 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:39:01.038 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:01.035+0000 7feb9c57dd80 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x5635ae64f0c0 0x5635ae64ae50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:39:01.038 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:01.035+0000 7feb9c57dd80 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x5635ae64b390 0x5635ae63f9f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:39:01.038 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:01.035+0000 7feb9c57dd80 1 -- --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x5635ae651cb0 con 0x5635ae646e20 2024-09-14T03:39:01.039 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:01.035+0000 7feb9c57dd80 1 -- --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x5635ae436540 con 0x5635ae64f0c0 2024-09-14T03:39:01.039 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:01.035+0000 7feb9c57dd80 1 -- --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x5635ae651e30 con 0x5635ae64b390 2024-09-14T03:39:01.039 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:01.035+0000 7feb92872640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x5635ae64f0c0 0x5635ae64ae50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:39:01.039 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:01.035+0000 7feb9b073640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x5635ae646e20 0x5635ae64a910 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:39:01.039 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:01.035+0000 7feb92872640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x5635ae64f0c0 0x5635ae64ae50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3301/0 says I am v2:172.21.15.83:33958/0 (socket says 172.21.15.83:33958) 2024-09-14T03:39:01.039 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:01.035+0000 7feb9c2d5640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x5635ae64b390 0x5635ae63f9f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:39:01.039 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:01.035+0000 7feb92872640 1 -- 172.21.15.83:0/285685727 learned_addr learned my addr 172.21.15.83:0/285685727 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:39:01.039 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:01.035+0000 7feb9b073640 1 -- 172.21.15.83:0/285685727 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x5635ae64b390 msgr2=0x5635ae63f9f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:01.039 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:01.035+0000 7feb9b073640 1 --2- 172.21.15.83:0/285685727 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x5635ae64b390 0x5635ae63f9f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:01.039 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:01.035+0000 7feb9b073640 1 -- 172.21.15.83:0/285685727 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x5635ae64f0c0 msgr2=0x5635ae64ae50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:01.039 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:01.035+0000 7feb9b073640 1 --2- 172.21.15.83:0/285685727 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x5635ae64f0c0 0x5635ae64ae50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:01.039 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:01.035+0000 7feb9b073640 1 -- 172.21.15.83:0/285685727 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5635ae649e50 con 0x5635ae646e20 2024-09-14T03:39:01.039 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:01.035+0000 7feb9c2d5640 1 --2- 172.21.15.83:0/285685727 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x5635ae64b390 0x5635ae63f9f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:39:01.039 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:01.035+0000 7feb92872640 1 --2- 172.21.15.83:0/285685727 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x5635ae64f0c0 0x5635ae64ae50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:39:01.040 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:01.035+0000 7feb9b073640 1 --2- 172.21.15.83:0/285685727 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x5635ae646e20 0x5635ae64a910 secure :-1 s=READY pgs=56 cs=0 l=1 rev1=1 crypto rx=0x7feb8c00bc70 tx=0x7feb8c009510 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:39:01.040 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:01.035+0000 7feb93fff640 1 -- 172.21.15.83:0/285685727 <== mon.0 v2:172.21.15.83:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7feb8c009e30 con 0x5635ae646e20 2024-09-14T03:39:01.040 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:01.036+0000 7feb93fff640 1 -- 172.21.15.83:0/285685727 <== mon.0 v2:172.21.15.83:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7feb8c010440 con 0x5635ae646e20 2024-09-14T03:39:01.040 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:01.036+0000 7feb9c57dd80 1 -- 172.21.15.83:0/285685727 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x5635ae64bcf0 con 0x5635ae646e20 2024-09-14T03:39:01.040 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:01.036+0000 7feb93fff640 1 -- 172.21.15.83:0/285685727 <== mon.0 v2:172.21.15.83:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7feb8c010890 con 0x5635ae646e20 2024-09-14T03:39:01.040 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:01.036+0000 7feb9c57dd80 1 -- 172.21.15.83:0/285685727 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5635ae64c930 con 0x5635ae646e20 2024-09-14T03:39:01.040 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:01.036+0000 7feb93fff640 1 -- 172.21.15.83:0/285685727 <== mon.0 v2:172.21.15.83:3300/0 4 ==== mgrmap(e 4) ==== 55943+0+0 (secure 0 0 0) 0x7feb8c018020 con 0x5635ae646e20 2024-09-14T03:39:01.040 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:01.037+0000 7feb93fff640 1 -- 172.21.15.83:0/285685727 <== mon.0 v2:172.21.15.83:3300/0 5 ==== osd_map(15..15 src has 1..15) ==== 2801+0+0 (secure 0 0 0) 0x7feb8c05be80 con 0x5635ae646e20 2024-09-14T03:39:01.040 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:01.037+0000 7feb9c57dd80 1 -- 172.21.15.83:0/285685727 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_command({"prefix": "osd pool application enable","pool": "rbd","app": "rbd"} v 0) -- 0x5635ae640260 con 0x5635ae646e20 2024-09-14T03:39:01.800 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:01.799+0000 7feb93fff640 1 -- 172.21.15.83:0/285685727 <== mon.0 v2:172.21.15.83:3300/0 6 ==== mon_command_ack([{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]=0 enabled application 'rbd' on pool 'rbd' v16) ==== 141+0+0 (secure 0 0 0) 0x7feb8c03b1c0 con 0x5635ae646e20 2024-09-14T03:39:01.801 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:01.799+0000 7feb9c57dd80 1 -- 172.21.15.83:0/285685727 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_get_version(what=osdmap handle=1) -- 0x7feb94051070 con 0x5635ae646e20 2024-09-14T03:39:01.808 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:01.807+0000 7feb93fff640 1 -- 172.21.15.83:0/285685727 <== mon.0 v2:172.21.15.83:3300/0 7 ==== mgrmap(e 5) ==== 55999+0+0 (secure 0 0 0) 0x7feb8c05abc0 con 0x5635ae646e20 2024-09-14T03:39:01.808 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:01.807+0000 7feb93fff640 1 -- 172.21.15.83:0/285685727 <== mon.0 v2:172.21.15.83:3300/0 8 ==== mon_get_version_reply(handle=1 version=16) ==== 24+0+0 (secure 0 0 0) 0x7feb8c05e6a0 con 0x5635ae646e20 2024-09-14T03:39:01.808 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:01.807+0000 7feb9a071640 1 -- 172.21.15.83:0/285685727 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({osdmap=16}) -- 0x7feb80002ae0 con 0x5635ae646e20 2024-09-14T03:39:01.809 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:01.807+0000 7feb93fff640 1 -- 172.21.15.83:0/285685727 <== mon.0 v2:172.21.15.83:3300/0 9 ==== osd_map(16..16 src has 1..16) ==== 650+0+0 (secure 0 0 0) 0x7feb8c009e30 con 0x5635ae646e20 2024-09-14T03:39:01.809 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:01.808+0000 7feb9c57dd80 1 -- 172.21.15.83:0/285685727 --> v1:172.21.15.83:6809/2350261690 -- osd_op(unknown.0.0:1 2.3 2:c4c92e5a:::rbd_trash:head [create] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e16) -- 0x5635ae6683a0 con 0x5635ae5a19b0 2024-09-14T03:39:01.811 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:01.810+0000 7feb9c2d5640 1 -- 172.21.15.83:0/285685727 <== osd.3 v1:172.21.15.83:6809/2350261690 1 ==== osd_op_reply(1 rbd_trash [create] v16'1 uv1 ondisk = 0) ==== 153+0+0 (unknown 2685446591 0 0) 0x7feb9400b040 con 0x5635ae5a19b0 2024-09-14T03:39:01.814 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:01.812+0000 7feb9c57dd80 1 -- 172.21.15.83:0/285685727 --> v1:172.21.15.83:6809/2350261690 -- osd_op(unknown.0.0:2 2.2 2:5cea7035:::rbd_info:head [call rbd.metadata_list in=33b] snapc 0=[] ondisk+read+known_if_redirected+supports_pool_eio e16) -- 0x5635ae6ba240 con 0x5635ae5a19b0 2024-09-14T03:39:01.814 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:01.813+0000 7feb9c2d5640 1 -- 172.21.15.83:0/285685727 <== osd.3 v1:172.21.15.83:6809/2350261690 2 ==== osd_op_reply(2 rbd_info [call] v0'0 uv0 ondisk = -2 ((2) No such file or directory)) ==== 152+0+0 (unknown 2026326479 0 0) 0x7feb9400b040 con 0x5635ae5a19b0 2024-09-14T03:39:01.814 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:01.813+0000 7feb9c57dd80 1 -- 172.21.15.83:0/285685727 --> v1:172.21.15.83:6809/2350261690 -- osd_op(unknown.0.0:3 2.2 2:5cea7035:::rbd_info:head [read 0~0] snapc 0=[] ondisk+read+known_if_redirected+supports_pool_eio e16) -- 0x5635ae6b9330 con 0x5635ae5a19b0 2024-09-14T03:39:01.814 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:01.813+0000 7feb9c2d5640 1 -- 172.21.15.83:0/285685727 <== osd.3 v1:172.21.15.83:6809/2350261690 3 ==== osd_op_reply(3 rbd_info [read 0~0] v0'0 uv0 ondisk = -2 ((2) No such file or directory)) ==== 152+0+0 (unknown 2039985091 0 0) 0x7feb9400b040 con 0x5635ae5a19b0 2024-09-14T03:39:01.814 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:01.813+0000 7feb9a071640 1 -- 172.21.15.83:0/285685727 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- pool_op(create unmanaged snap pool 2 tid 4 name v0) -- 0x7feb80003f20 con 0x5635ae646e20 2024-09-14T03:39:02.809 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:02.807+0000 7feb93fff640 1 -- 172.21.15.83:0/285685727 <== mon.0 v2:172.21.15.83:3300/0 10 ==== pool_op_reply(tid 4 (0) Success v17) ==== 55+0+0 (secure 0 0 0) 0x7feb8c03b420 con 0x5635ae646e20 2024-09-14T03:39:02.809 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:02.807+0000 7feb93fff640 1 -- 172.21.15.83:0/285685727 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({osdmap=17}) -- 0x7feb7c04cfe0 con 0x5635ae646e20 2024-09-14T03:39:02.809 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:02.807+0000 7feb93fff640 1 -- 172.21.15.83:0/285685727 <== mon.0 v2:172.21.15.83:3300/0 11 ==== osd_map(17..17 src has 1..17) ==== 626+0+0 (secure 0 0 0) 0x7feb8c039790 con 0x5635ae646e20 2024-09-14T03:39:02.809 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:02.808+0000 7feb9a071640 1 -- 172.21.15.83:0/285685727 --> v1:172.21.15.83:6809/2350261690 -- osd_op(unknown.0.0:5 2.2 2:5cea7035:::rbd_info:head [create,write 0~8 in=8b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e17) -- 0x7feb80005920 con 0x5635ae5a19b0 2024-09-14T03:39:02.811 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:02.809+0000 7feb9c2d5640 1 -- 172.21.15.83:0/285685727 <== osd.3 v1:172.21.15.83:6809/2350261690 4 ==== osd_op_reply(5 rbd_info [create,write 0~8] v17'1 uv1 ondisk = 0) ==== 194+0+0 (unknown 3115756469 0 0) 0x7feb9400b040 con 0x5635ae5a19b0 2024-09-14T03:39:02.811 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:02.809+0000 7feb9a071640 1 -- 172.21.15.83:0/285685727 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- pool_op(delete unmanaged snap pool 2 tid 6 name v17) -- 0x7feb80005470 con 0x5635ae646e20 2024-09-14T03:39:03.813 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:03.811+0000 7feb93fff640 1 -- 172.21.15.83:0/285685727 <== mon.0 v2:172.21.15.83:3300/0 12 ==== pool_op_reply(tid 6 (0) Success v18) ==== 43+0+0 (secure 0 0 0) 0x7feb8c037210 con 0x5635ae646e20 2024-09-14T03:39:03.813 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:03.811+0000 7feb93fff640 1 -- 172.21.15.83:0/285685727 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({osdmap=18}) -- 0x7feb7c04d1b0 con 0x5635ae646e20 2024-09-14T03:39:03.813 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:03.812+0000 7feb93fff640 1 -- 172.21.15.83:0/285685727 <== mon.0 v2:172.21.15.83:3300/0 13 ==== osd_map(18..18 src has 1..18) ==== 654+0+0 (secure 0 0 0) 0x7feb8c0452c0 con 0x5635ae646e20 2024-09-14T03:39:03.813 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:03.812+0000 7feb9a071640 1 -- 172.21.15.83:0/285685727 --> v1:172.21.15.83:6809/2350261690 -- osd_op(unknown.0.0:7 2.2 2:5cea7035:::rbd_info:head [write 0~19 in=19b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e18) -- 0x7feb80006170 con 0x5635ae5a19b0 2024-09-14T03:39:03.816 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:03.815+0000 7feb9c2d5640 1 -- 172.21.15.83:0/285685727 <== osd.3 v1:172.21.15.83:6809/2350261690 5 ==== osd_op_reply(7 rbd_info [write 0~19] v18'2 uv2 ondisk = 0) ==== 152+0+0 (unknown 4030416863 0 0) 0x7feb9400b040 con 0x5635ae5a19b0 2024-09-14T03:39:03.817 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:03.815+0000 7feb9c57dd80 1 -- 172.21.15.83:0/285685727 >> v1:172.21.15.83:6809/2350261690 conn(0x5635ae5a19b0 legacy=0x5635ae666c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:03.817 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:03.815+0000 7feb9c57dd80 1 -- 172.21.15.83:0/285685727 >> v1:172.21.15.83:6800/1934165865 conn(0x7feb7c046830 legacy=0x7feb7c048cf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:03.817 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:03.815+0000 7feb9c57dd80 1 -- 172.21.15.83:0/285685727 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x5635ae646e20 msgr2=0x5635ae64a910 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:03.817 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:03.815+0000 7feb9c57dd80 1 --2- 172.21.15.83:0/285685727 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x5635ae646e20 0x5635ae64a910 secure :-1 s=READY pgs=56 cs=0 l=1 rev1=1 crypto rx=0x7feb8c00bc70 tx=0x7feb8c009510 comp rx=0 tx=0).stop 2024-09-14T03:39:03.818 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:03.815+0000 7feb9c2d5640 1 -- 172.21.15.83:0/285685727 reap_dead start 2024-09-14T03:39:03.818 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:03.815+0000 7feb9c57dd80 1 -- 172.21.15.83:0/285685727 shutdown_connections 2024-09-14T03:39:03.818 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:03.815+0000 7feb9c57dd80 1 -- 172.21.15.83:0/285685727 >> 172.21.15.83:0/285685727 conn(0x5635ae5b9060 msgr2=0x5635ae5b6ae0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:39:03.818 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:03.816+0000 7feb9c57dd80 1 -- 172.21.15.83:0/285685727 shutdown_connections 2024-09-14T03:39:03.818 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:03.816+0000 7feb9c57dd80 1 -- 172.21.15.83:0/285685727 wait complete. 2024-09-14T03:39:03.821 INFO:tasks.ceph:Starting mds daemons in cluster ceph... 2024-09-14T03:39:03.822 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph config log 1 --format=json 2024-09-14T03:39:03.822 INFO:tasks.daemonwatchdog.daemon_watchdog:watchdog starting 2024-09-14T03:39:03.932 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:03.930+0000 7f0680583640 1 -- 172.21.15.83:0/4060781234 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f067806e070 msgr2=0x7f067806e490 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:03.932 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:03.930+0000 7f0680583640 1 --2- 172.21.15.83:0/4060781234 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f067806e070 0x7f067806e490 secure :-1 s=READY pgs=34 cs=0 l=1 rev1=1 crypto rx=0x7f0668009830 tx=0x7f066802a360 comp rx=0 tx=0).stop 2024-09-14T03:39:03.932 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:03.930+0000 7f0680583640 1 -- 172.21.15.83:0/4060781234 shutdown_connections 2024-09-14T03:39:03.932 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:03.930+0000 7f0680583640 1 --2- 172.21.15.83:0/4060781234 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f067806e070 0x7f067806e490 unknown :-1 s=CLOSED pgs=34 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:03.932 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:03.930+0000 7f0680583640 1 --2- 172.21.15.83:0/4060781234 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f06781324a0 0x7f0678134890 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:03.933 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:03.930+0000 7f0680583640 1 --2- 172.21.15.83:0/4060781234 >> v2:172.21.15.83:3300/0 conn(0x7f067806d750 0x7f067806db30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:03.933 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:03.930+0000 7f0680583640 1 -- 172.21.15.83:0/4060781234 >> 172.21.15.83:0/4060781234 conn(0x7f067809c630 msgr2=0x7f067809ca40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:39:03.933 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:03.930+0000 7f0680583640 1 -- 172.21.15.83:0/4060781234 shutdown_connections 2024-09-14T03:39:03.933 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:03.930+0000 7f0680583640 1 -- 172.21.15.83:0/4060781234 wait complete. 2024-09-14T03:39:03.935 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:03.931+0000 7f0680583640 1 Processor -- start 2024-09-14T03:39:03.935 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:03.931+0000 7f0680583640 1 -- start start 2024-09-14T03:39:03.935 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:03.932+0000 7f0680583640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f067806d750 0x7f0678130c00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:39:03.935 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:03.932+0000 7f0680583640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f067806e070 0x7f0678131140 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:39:03.935 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:03.932+0000 7f0680583640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f06781324a0 0x7f0678131680 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:39:03.935 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:03.932+0000 7f0680583640 1 -- --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7f0678135b90 con 0x7f06781324a0 2024-09-14T03:39:03.936 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:03.932+0000 7f0680583640 1 -- --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7f0678135a10 con 0x7f067806e070 2024-09-14T03:39:03.936 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:03.932+0000 7f0680583640 1 -- --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7f0678135d10 con 0x7f067806d750 2024-09-14T03:39:03.936 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:03.932+0000 7f067e31f640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f067806d750 0x7f0678130c00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:39:03.936 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:03.932+0000 7f067e31f640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f067806d750 0x7f0678130c00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3302/0 says I am v2:172.21.15.83:60934/0 (socket says 172.21.15.83:60934) 2024-09-14T03:39:03.936 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:03.932+0000 7f067e31f640 1 -- 172.21.15.83:0/1316086971 learned_addr learned my addr 172.21.15.83:0/1316086971 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:39:03.936 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:03.932+0000 7f067eb20640 1 --2- 172.21.15.83:0/1316086971 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f06781324a0 0x7f0678131680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:39:03.936 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:03.932+0000 7f067db1e640 1 --2- 172.21.15.83:0/1316086971 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f067806e070 0x7f0678131140 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:39:03.936 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:03.932+0000 7f067e31f640 1 -- 172.21.15.83:0/1316086971 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f067806e070 msgr2=0x7f0678131140 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:03.936 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:03.932+0000 7f067e31f640 1 --2- 172.21.15.83:0/1316086971 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f067806e070 0x7f0678131140 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:03.936 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:03.932+0000 7f067e31f640 1 -- 172.21.15.83:0/1316086971 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f06781324a0 msgr2=0x7f0678131680 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:03.936 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:03.932+0000 7f067e31f640 1 --2- 172.21.15.83:0/1316086971 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f06781324a0 0x7f0678131680 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:03.937 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:03.932+0000 7f067e31f640 1 -- 172.21.15.83:0/1316086971 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f06781247f0 con 0x7f067806d750 2024-09-14T03:39:03.937 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:03.932+0000 7f067db1e640 1 --2- 172.21.15.83:0/1316086971 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f067806e070 0x7f0678131140 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:39:03.937 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:03.932+0000 7f067eb20640 1 --2- 172.21.15.83:0/1316086971 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f06781324a0 0x7f0678131680 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:39:03.937 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:03.932+0000 7f067e31f640 1 --2- 172.21.15.83:0/1316086971 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f067806d750 0x7f0678130c00 secure :-1 s=READY pgs=37 cs=0 l=1 rev1=1 crypto rx=0x7f067400e9c0 tx=0x7f067400ee80 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:39:03.937 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:03.932+0000 7f066effd640 1 -- 172.21.15.83:0/1316086971 <== mon.2 v2:172.21.15.83:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f067400ccc0 con 0x7f067806d750 2024-09-14T03:39:03.937 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:03.932+0000 7f066effd640 1 -- 172.21.15.83:0/1316086971 <== mon.2 v2:172.21.15.83:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f0674005020 con 0x7f067806d750 2024-09-14T03:39:03.937 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:03.932+0000 7f0680583640 1 -- 172.21.15.83:0/1316086971 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f06780737d0 con 0x7f067806d750 2024-09-14T03:39:03.937 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:03.932+0000 7f066effd640 1 -- 172.21.15.83:0/1316086971 <== mon.2 v2:172.21.15.83:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f067400ce60 con 0x7f067806d750 2024-09-14T03:39:03.937 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:03.932+0000 7f0680583640 1 -- 172.21.15.83:0/1316086971 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7f06780719c0 con 0x7f067806d750 2024-09-14T03:39:03.937 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:03.933+0000 7f066effd640 1 -- 172.21.15.83:0/1316086971 <== mon.2 v2:172.21.15.83:3302/0 4 ==== mgrmap(e 5) ==== 55999+0+0 (secure 0 0 0) 0x7f0674005450 con 0x7f067806d750 2024-09-14T03:39:03.937 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:03.933+0000 7f0680583640 1 -- 172.21.15.83:0/1316086971 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f067806db30 con 0x7f067806d750 2024-09-14T03:39:03.938 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:03.934+0000 7f066effd640 1 -- 172.21.15.83:0/1316086971 <== mon.2 v2:172.21.15.83:3302/0 5 ==== osd_map(18..18 src has 1..18) ==== 2868+0+0 (secure 0 0 0) 0x7f067405e230 con 0x7f067806d750 2024-09-14T03:39:03.939 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:03.938+0000 7f066effd640 1 -- 172.21.15.83:0/1316086971 <== mon.2 v2:172.21.15.83:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+151260 (secure 0 0 0) 0x7f067405d380 con 0x7f067806d750 2024-09-14T03:39:04.082 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.080+0000 7f0680583640 1 -- 172.21.15.83:0/1316086971 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_command({"prefix": "config log", "num": 1, "format": "json"} v 0) -- 0x7f06780952e0 con 0x7f067806d750 2024-09-14T03:39:04.082 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.081+0000 7f066effd640 1 -- 172.21.15.83:0/1316086971 <== mon.2 v2:172.21.15.83:3302/0 7 ==== mon_command_ack([{"prefix": "config log", "num": 1, "format": "json"}]=0 v23) ==== 86+0+160 (secure 0 0 0) 0x7f067403d4c0 con 0x7f067806d750 2024-09-14T03:39:04.082 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:39:04.084 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.082+0000 7f0680583640 1 -- 172.21.15.83:0/1316086971 >> v1:172.21.15.83:6800/1934165865 conn(0x7f065c047fd0 legacy=0x7f065c04a490 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:04.084 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.082+0000 7f0680583640 1 -- 172.21.15.83:0/1316086971 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f067806d750 msgr2=0x7f0678130c00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:04.084 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.082+0000 7f0680583640 1 --2- 172.21.15.83:0/1316086971 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f067806d750 0x7f0678130c00 secure :-1 s=READY pgs=37 cs=0 l=1 rev1=1 crypto rx=0x7f067400e9c0 tx=0x7f067400ee80 comp rx=0 tx=0).stop 2024-09-14T03:39:04.084 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.082+0000 7f0680583640 1 -- 172.21.15.83:0/1316086971 shutdown_connections 2024-09-14T03:39:04.085 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.082+0000 7f0680583640 1 --2- 172.21.15.83:0/1316086971 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f06781324a0 0x7f0678131680 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:04.085 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.082+0000 7f0680583640 1 --2- 172.21.15.83:0/1316086971 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f067806e070 0x7f0678131140 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:04.085 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.082+0000 7f0680583640 1 --2- 172.21.15.83:0/1316086971 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f067806d750 0x7f0678130c00 unknown :-1 s=CLOSED pgs=37 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:04.085 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.082+0000 7f0680583640 1 -- 172.21.15.83:0/1316086971 >> 172.21.15.83:0/1316086971 conn(0x7f067809c630 msgr2=0x7f0678099cb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:39:04.085 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.082+0000 7f0680583640 1 -- 172.21.15.83:0/1316086971 shutdown_connections 2024-09-14T03:39:04.085 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.082+0000 7f0680583640 1 -- 172.21.15.83:0/1316086971 wait complete. 2024-09-14T03:39:04.095 INFO:teuthology.orchestra.run.smithi083.stdout:[{"version":23,"timestamp":"2024-09-14T03:38:55.820206+0000","name":"","changes":[{"name":"osd.2/osd_mclock_max_capacity_iops_ssd","new_value":"5942.322557"}]}] 2024-09-14T03:39:04.095 INFO:tasks.ceph_manager:config epoch is 23 2024-09-14T03:39:04.095 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-09-14T03:39:04.095 INFO:tasks.ceph.ceph_manager.ceph:waiting for mgr available 2024-09-14T03:39:04.095 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph mgr dump --format=json 2024-09-14T03:39:04.195 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.193+0000 7f3f622d7640 1 -- 172.21.15.83:0/3686326937 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f3f5c093d70 msgr2=0x7f3f5c094150 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:04.195 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.193+0000 7f3f622d7640 1 --2- 172.21.15.83:0/3686326937 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f3f5c093d70 0x7f3f5c094150 secure :-1 s=READY pgs=57 cs=0 l=1 rev1=1 crypto rx=0x7f3f4c009830 tx=0x7f3f4c02a350 comp rx=0 tx=0).stop 2024-09-14T03:39:04.196 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.193+0000 7f3f622d7640 1 -- 172.21.15.83:0/3686326937 shutdown_connections 2024-09-14T03:39:04.196 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.193+0000 7f3f622d7640 1 --2- 172.21.15.83:0/3686326937 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f3f5c092070 0x7f3f5c092500 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:04.196 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.193+0000 7f3f622d7640 1 --2- 172.21.15.83:0/3686326937 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f3f5c095fc0 0x7f3f5c094690 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:04.196 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.193+0000 7f3f622d7640 1 --2- 172.21.15.83:0/3686326937 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f3f5c093d70 0x7f3f5c094150 unknown :-1 s=CLOSED pgs=57 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:04.196 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.193+0000 7f3f622d7640 1 -- 172.21.15.83:0/3686326937 >> 172.21.15.83:0/3686326937 conn(0x7f3f5c099450 msgr2=0x7f3f5c0969e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:39:04.196 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.193+0000 7f3f622d7640 1 -- 172.21.15.83:0/3686326937 shutdown_connections 2024-09-14T03:39:04.196 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.194+0000 7f3f622d7640 1 -- 172.21.15.83:0/3686326937 wait complete. 2024-09-14T03:39:04.199 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.194+0000 7f3f622d7640 1 Processor -- start 2024-09-14T03:39:04.200 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.195+0000 7f3f622d7640 1 -- start start 2024-09-14T03:39:04.200 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.195+0000 7f3f622d7640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f3f5c092070 0x7f3f5c130350 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:39:04.200 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.195+0000 7f3f622d7640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f3f5c093d70 0x7f3f5c090a60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:39:04.200 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.195+0000 7f3f622d7640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f3f5c095fc0 0x7f3f5c090fa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:39:04.200 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.195+0000 7f3f622d7640 1 -- --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7f3f5c093ac0 con 0x7f3f5c093d70 2024-09-14T03:39:04.200 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.195+0000 7f3f622d7640 1 -- --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7f3f5c093940 con 0x7f3f5c092070 2024-09-14T03:39:04.200 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.195+0000 7f3f622d7640 1 -- --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7f3f5c1317e0 con 0x7f3f5c095fc0 2024-09-14T03:39:04.200 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.195+0000 7f3f5bfff640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f3f5c092070 0x7f3f5c130350 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:39:04.201 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.195+0000 7f3f5b7fe640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f3f5c093d70 0x7f3f5c090a60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:39:04.201 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.195+0000 7f3f5bfff640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f3f5c092070 0x7f3f5c130350 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3301/0 says I am v2:172.21.15.83:60424/0 (socket says 172.21.15.83:60424) 2024-09-14T03:39:04.201 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.195+0000 7f3f5bfff640 1 -- 172.21.15.83:0/4039096954 learned_addr learned my addr 172.21.15.83:0/4039096954 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:39:04.201 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.195+0000 7f3f60874640 1 --2- 172.21.15.83:0/4039096954 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f3f5c095fc0 0x7f3f5c090fa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:39:04.201 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.195+0000 7f3f5bfff640 1 -- 172.21.15.83:0/4039096954 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f3f5c095fc0 msgr2=0x7f3f5c090fa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:04.201 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.195+0000 7f3f5bfff640 1 --2- 172.21.15.83:0/4039096954 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f3f5c095fc0 0x7f3f5c090fa0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:04.201 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.195+0000 7f3f5bfff640 1 -- 172.21.15.83:0/4039096954 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f3f5c093d70 msgr2=0x7f3f5c090a60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:04.201 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.195+0000 7f3f5bfff640 1 --2- 172.21.15.83:0/4039096954 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f3f5c093d70 0x7f3f5c090a60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:04.202 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.195+0000 7f3f5bfff640 1 -- 172.21.15.83:0/4039096954 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f3f5c08f7a0 con 0x7f3f5c092070 2024-09-14T03:39:04.202 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.195+0000 7f3f5b7fe640 1 --2- 172.21.15.83:0/4039096954 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f3f5c093d70 0x7f3f5c090a60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:39:04.202 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.195+0000 7f3f60874640 1 --2- 172.21.15.83:0/4039096954 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f3f5c095fc0 0x7f3f5c090fa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:39:04.202 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.195+0000 7f3f5bfff640 1 --2- 172.21.15.83:0/4039096954 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f3f5c092070 0x7f3f5c130350 secure :-1 s=READY pgs=35 cs=0 l=1 rev1=1 crypto rx=0x7f3f4c000c00 tx=0x7f3f4c033af0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:39:04.202 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.196+0000 7f3f58ff9640 1 -- 172.21.15.83:0/4039096954 <== mon.1 v2:172.21.15.83:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3f4c03f020 con 0x7f3f5c092070 2024-09-14T03:39:04.202 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.196+0000 7f3f58ff9640 1 -- 172.21.15.83:0/4039096954 <== mon.1 v2:172.21.15.83:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f3f4c033ce0 con 0x7f3f5c092070 2024-09-14T03:39:04.202 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.196+0000 7f3f622d7640 1 -- 172.21.15.83:0/4039096954 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f3f5c08fbf0 con 0x7f3f5c092070 2024-09-14T03:39:04.202 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.196+0000 7f3f58ff9640 1 -- 172.21.15.83:0/4039096954 <== mon.1 v2:172.21.15.83:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3f4c037040 con 0x7f3f5c092070 2024-09-14T03:39:04.202 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.196+0000 7f3f622d7640 1 -- 172.21.15.83:0/4039096954 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7f3f5c133aa0 con 0x7f3f5c092070 2024-09-14T03:39:04.202 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.197+0000 7f3f622d7640 1 -- 172.21.15.83:0/4039096954 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f3f5c09af10 con 0x7f3f5c092070 2024-09-14T03:39:04.202 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.197+0000 7f3f58ff9640 1 -- 172.21.15.83:0/4039096954 <== mon.1 v2:172.21.15.83:3301/0 4 ==== mgrmap(e 5) ==== 55999+0+0 (secure 0 0 0) 0x7f3f4c042020 con 0x7f3f5c092070 2024-09-14T03:39:04.202 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.197+0000 7f3f58ff9640 1 -- 172.21.15.83:0/4039096954 <== mon.1 v2:172.21.15.83:3301/0 5 ==== osd_map(18..18 src has 1..18) ==== 2868+0+0 (secure 0 0 0) 0x7f3f4c07d770 con 0x7f3f5c092070 2024-09-14T03:39:04.203 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.201+0000 7f3f58ff9640 1 -- 172.21.15.83:0/4039096954 <== mon.1 v2:172.21.15.83:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+151260 (secure 0 0 0) 0x7f3f5c09af10 con 0x7f3f5c092070 2024-09-14T03:39:04.369 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.367+0000 7f3f622d7640 1 -- 172.21.15.83:0/4039096954 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_command({"prefix": "mgr dump", "format": "json"} v 0) -- 0x7f3f5c1313f0 con 0x7f3f5c092070 2024-09-14T03:39:04.371 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.369+0000 7f3f58ff9640 1 -- 172.21.15.83:0/4039096954 <== mon.1 v2:172.21.15.83:3301/0 7 ==== mon_command_ack([{"prefix": "mgr dump", "format": "json"}]=0 v5) ==== 74+0+108900 (secure 0 0 0) 0x7f3f5c1313f0 con 0x7f3f5c092070 2024-09-14T03:39:04.373 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:39:04.374 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.371+0000 7f3f622d7640 1 -- 172.21.15.83:0/4039096954 >> v1:172.21.15.83:6800/1934165865 conn(0x7f3f48048000 legacy=0x7f3f4804a4c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:04.374 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.371+0000 7f3f622d7640 1 -- 172.21.15.83:0/4039096954 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f3f5c092070 msgr2=0x7f3f5c130350 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:04.374 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.371+0000 7f3f622d7640 1 --2- 172.21.15.83:0/4039096954 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f3f5c092070 0x7f3f5c130350 secure :-1 s=READY pgs=35 cs=0 l=1 rev1=1 crypto rx=0x7f3f4c000c00 tx=0x7f3f4c033af0 comp rx=0 tx=0).stop 2024-09-14T03:39:04.374 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.371+0000 7f3f622d7640 1 -- 172.21.15.83:0/4039096954 shutdown_connections 2024-09-14T03:39:04.374 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.371+0000 7f3f622d7640 1 --2- 172.21.15.83:0/4039096954 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f3f5c095fc0 0x7f3f5c090fa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:04.375 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.371+0000 7f3f622d7640 1 --2- 172.21.15.83:0/4039096954 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f3f5c093d70 0x7f3f5c090a60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:04.375 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.371+0000 7f3f622d7640 1 --2- 172.21.15.83:0/4039096954 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f3f5c092070 0x7f3f5c130350 unknown :-1 s=CLOSED pgs=35 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:04.375 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.371+0000 7f3f622d7640 1 -- 172.21.15.83:0/4039096954 >> 172.21.15.83:0/4039096954 conn(0x7f3f5c099450 msgr2=0x7f3f5c099de0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:39:04.375 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.372+0000 7f3f622d7640 1 -- 172.21.15.83:0/4039096954 shutdown_connections 2024-09-14T03:39:04.375 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.372+0000 7f3f622d7640 1 -- 172.21.15.83:0/4039096954 wait complete. 2024-09-14T03:39:04.384 INFO:teuthology.orchestra.run.smithi083.stdout:{"epoch":5,"flags":0,"active_gid":4146,"active_name":"y","active_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6800","nonce":1934165865}]},"active_addr":"172.21.15.83:6800/1934165865","active_change":"2024-09-14T03:38:46.888221+0000","active_mgr_features":4540701547738038271,"available":true,"standbys":[{"gid":4099,"name":"x","mgr_features":4540701547738038271,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"secondary_zone_period_retry_limit":{"name":"secondary_zone_period_retry_limit","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"RGW module period update retry limit for secondary site","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["iostat","nfs","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"secondary_zone_period_retry_limit":{"name":"secondary_zone_period_retry_limit","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"RGW module period update retry limit for secondary site","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"squid":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":0,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.83:0","nonce":4246762177}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.83:0","nonce":293392365}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.83:0","nonce":3623382676}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.83:0","nonce":4208089382}]}]} 2024-09-14T03:39:04.387 INFO:tasks.ceph.ceph_manager.ceph:mgr available! 2024-09-14T03:39:04.387 INFO:tasks.ceph.ceph_manager.ceph:waiting for all up 2024-09-14T03:39:04.387 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-09-14T03:39:04.489 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.485+0000 7fecbdf45640 1 -- 172.21.15.83:0/2909805423 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fecb8097170 msgr2=0x7fecb8131060 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:04.489 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.485+0000 7fecbdf45640 1 --2- 172.21.15.83:0/2909805423 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fecb8097170 0x7fecb8131060 secure :-1 s=READY pgs=36 cs=0 l=1 rev1=1 crypto rx=0x7feca0009860 tx=0x7feca002a460 comp rx=0 tx=0).stop 2024-09-14T03:39:04.489 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.485+0000 7fecbdf45640 1 -- 172.21.15.83:0/2909805423 shutdown_connections 2024-09-14T03:39:04.489 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.485+0000 7fecbdf45640 1 --2- 172.21.15.83:0/2909805423 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fecb81315a0 0x7fecb8133990 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:04.489 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.485+0000 7fecbdf45640 1 --2- 172.21.15.83:0/2909805423 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fecb8097170 0x7fecb8131060 unknown :-1 s=CLOSED pgs=36 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:04.489 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.485+0000 7fecbdf45640 1 -- 172.21.15.83:0/2909805423 >> 172.21.15.83:0/2909805423 conn(0x7fecb809c6a0 msgr2=0x7fecb809cab0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:39:04.489 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.485+0000 7fecbdf45640 1 -- 172.21.15.83:0/2909805423 shutdown_connections 2024-09-14T03:39:04.489 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.485+0000 7fecbdf45640 1 -- 172.21.15.83:0/2909805423 wait complete. 2024-09-14T03:39:04.489 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.486+0000 7fecbdf45640 1 Processor -- start 2024-09-14T03:39:04.489 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.486+0000 7fecbdf45640 1 -- start start 2024-09-14T03:39:04.490 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.486+0000 7fecbdf45640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fecb8096680 0x7fecb8092350 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:39:04.490 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.486+0000 7fecbdf45640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fecb8097170 0x7fecb8092890 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:39:04.490 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.486+0000 7fecbdf45640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fecb81315a0 0x7fecb8092dd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:39:04.490 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.486+0000 7fecbdf45640 1 -- --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7fecb8134820 con 0x7fecb81315a0 2024-09-14T03:39:04.490 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.486+0000 7fecbdf45640 1 -- --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7fecb81346a0 con 0x7fecb8097170 2024-09-14T03:39:04.490 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.486+0000 7fecbdf45640 1 -- --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7fecb81349a0 con 0x7fecb8096680 2024-09-14T03:39:04.490 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.486+0000 7fecaffff640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fecb8097170 0x7fecb8092890 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:39:04.490 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.486+0000 7fecb77fe640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fecb8096680 0x7fecb8092350 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:39:04.490 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.486+0000 7fecb7fff640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fecb81315a0 0x7fecb8092dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:39:04.490 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.486+0000 7fecaffff640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fecb8097170 0x7fecb8092890 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3301/0 says I am v2:172.21.15.83:60438/0 (socket says 172.21.15.83:60438) 2024-09-14T03:39:04.490 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.486+0000 7fecaffff640 1 -- 172.21.15.83:0/1844927704 learned_addr learned my addr 172.21.15.83:0/1844927704 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:39:04.490 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.487+0000 7fecaffff640 1 -- 172.21.15.83:0/1844927704 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fecb8096680 msgr2=0x7fecb8092350 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:04.490 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.487+0000 7fecaffff640 1 --2- 172.21.15.83:0/1844927704 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fecb8096680 0x7fecb8092350 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:04.490 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.487+0000 7fecaffff640 1 -- 172.21.15.83:0/1844927704 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fecb81315a0 msgr2=0x7fecb8092dd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:04.491 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.487+0000 7fecaffff640 1 --2- 172.21.15.83:0/1844927704 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fecb81315a0 0x7fecb8092dd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:04.491 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.487+0000 7fecaffff640 1 -- 172.21.15.83:0/1844927704 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fecb806fb20 con 0x7fecb8097170 2024-09-14T03:39:04.491 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.487+0000 7fecb77fe640 1 --2- 172.21.15.83:0/1844927704 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fecb8096680 0x7fecb8092350 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:39:04.491 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.487+0000 7fecb7fff640 1 --2- 172.21.15.83:0/1844927704 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fecb81315a0 0x7fecb8092dd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:39:04.491 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.487+0000 7fecaffff640 1 --2- 172.21.15.83:0/1844927704 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fecb8097170 0x7fecb8092890 secure :-1 s=READY pgs=37 cs=0 l=1 rev1=1 crypto rx=0x7feca002aa50 tx=0x7feca0032c10 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:39:04.491 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.487+0000 7fecb4ff9640 1 -- 172.21.15.83:0/1844927704 <== mon.1 v2:172.21.15.83:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7feca0040070 con 0x7fecb8097170 2024-09-14T03:39:04.491 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.487+0000 7fecb4ff9640 1 -- 172.21.15.83:0/1844927704 <== mon.1 v2:172.21.15.83:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7feca0005310 con 0x7fecb8097170 2024-09-14T03:39:04.491 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.487+0000 7fecbdf45640 1 -- 172.21.15.83:0/1844927704 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fecb8093ea0 con 0x7fecb8097170 2024-09-14T03:39:04.491 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.487+0000 7fecb4ff9640 1 -- 172.21.15.83:0/1844927704 <== mon.1 v2:172.21.15.83:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7feca0005c90 con 0x7fecb8097170 2024-09-14T03:39:04.491 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.487+0000 7fecbdf45640 1 -- 172.21.15.83:0/1844927704 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7fecb8093310 con 0x7fecb8097170 2024-09-14T03:39:04.492 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.488+0000 7fecb4ff9640 1 -- 172.21.15.83:0/1844927704 <== mon.1 v2:172.21.15.83:3301/0 4 ==== mgrmap(e 5) ==== 55999+0+0 (secure 0 0 0) 0x7feca0039040 con 0x7fecb8097170 2024-09-14T03:39:04.492 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.488+0000 7fecbdf45640 1 -- 172.21.15.83:0/1844927704 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fecb8096a60 con 0x7fecb8097170 2024-09-14T03:39:04.492 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.488+0000 7fecb4ff9640 1 -- 172.21.15.83:0/1844927704 <== mon.1 v2:172.21.15.83:3301/0 5 ==== osd_map(18..18 src has 1..18) ==== 2868+0+0 (secure 0 0 0) 0x7feca007e830 con 0x7fecb8097170 2024-09-14T03:39:04.493 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.492+0000 7fecb4ff9640 1 -- 172.21.15.83:0/1844927704 <== mon.1 v2:172.21.15.83:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+151260 (secure 0 0 0) 0x7feca007d980 con 0x7fecb8097170 2024-09-14T03:39:04.633 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.631+0000 7fecbdf45640 1 -- 172.21.15.83:0/1844927704 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) -- 0x7fecb8094670 con 0x7fecb8097170 2024-09-14T03:39:04.633 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.632+0000 7fecb4ff9640 1 -- 172.21.15.83:0/1844927704 <== mon.1 v2:172.21.15.83:3301/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v18) ==== 74+0+8917 (secure 0 0 0) 0x7feca007ee20 con 0x7fecb8097170 2024-09-14T03:39:04.633 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:39:04.634 INFO:teuthology.orchestra.run.smithi083.stdout:{"epoch":18,"fsid":"765298ef-df4c-4f28-af02-b45f60264ff9","created":"2024-09-14T03:38:43.954952+0000","modified":"2024-09-14T03:39:03.805107+0000","last_up_change":"2024-09-14T03:38:56.772236+0000","last_in_change":"2024-09-14T03:38:46.006167+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-14T03:38:56.922548+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"14","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":4,"score_stable":4,"optimal_score":0.5,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-14T03:39:00.433011+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":18,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.5,"score_stable":2.5,"optimal_score":1,"raw_score_acting":2.5,"raw_score_stable":2.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ff477614-343d-4d97-9df5-94933f9b904d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6805","nonce":2271402019}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6806","nonce":2271402019}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6808","nonce":2271402019}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6807","nonce":2271402019}]},"public_addr":"172.21.15.83:6805/2271402019","cluster_addr":"172.21.15.83:6806/2271402019","heartbeat_back_addr":"172.21.15.83:6808/2271402019","heartbeat_front_addr":"172.21.15.83:6807/2271402019","state":["exists","up"]},{"osd":1,"uuid":"562ec9a0-3f91-4a29-896d-94340e9fa36f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":11,"up_thru":15,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6801","nonce":2493296278}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6802","nonce":2493296278}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6804","nonce":2493296278}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6803","nonce":2493296278}]},"public_addr":"172.21.15.83:6801/2493296278","cluster_addr":"172.21.15.83:6802/2493296278","heartbeat_back_addr":"172.21.15.83:6804/2493296278","heartbeat_front_addr":"172.21.15.83:6803/2493296278","state":["exists","up"]},{"osd":2,"uuid":"8c242dda-3308-4804-b2be-9a1737c42cf6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":11,"up_thru":15,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6810","nonce":1553686396}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6812","nonce":1553686396}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6816","nonce":1553686396}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6814","nonce":1553686396}]},"public_addr":"172.21.15.83:6810/1553686396","cluster_addr":"172.21.15.83:6812/1553686396","heartbeat_back_addr":"172.21.15.83:6816/1553686396","heartbeat_front_addr":"172.21.15.83:6814/1553686396","state":["exists","up"]},{"osd":3,"uuid":"c4dd2c36-3c86-4b56-86fb-fede641a436c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":11,"up_thru":15,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6809","nonce":2350261690}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6811","nonce":2350261690}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6815","nonce":2350261690}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6813","nonce":2350261690}]},"public_addr":"172.21.15.83:6809/2350261690","cluster_addr":"172.21.15.83:6811/2350261690","heartbeat_back_addr":"172.21.15.83:6815/2350261690","heartbeat_front_addr":"172.21.15.83:6813/2350261690","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:38:53.047098+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:38:53.354351+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:38:53.422284+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:38:53.620657+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-14T03:39:04.635 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.634+0000 7fecbdf45640 1 -- 172.21.15.83:0/1844927704 >> v1:172.21.15.83:6800/1934165865 conn(0x7fec98048000 legacy=0x7fec9804a4c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:04.635 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.634+0000 7fecbdf45640 1 -- 172.21.15.83:0/1844927704 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fecb8097170 msgr2=0x7fecb8092890 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:04.635 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.634+0000 7fecbdf45640 1 --2- 172.21.15.83:0/1844927704 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fecb8097170 0x7fecb8092890 secure :-1 s=READY pgs=37 cs=0 l=1 rev1=1 crypto rx=0x7feca002aa50 tx=0x7feca0032c10 comp rx=0 tx=0).stop 2024-09-14T03:39:04.635 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.634+0000 7fecbdf45640 1 -- 172.21.15.83:0/1844927704 shutdown_connections 2024-09-14T03:39:04.635 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.634+0000 7fecbdf45640 1 --2- 172.21.15.83:0/1844927704 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fecb81315a0 0x7fecb8092dd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:04.636 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.634+0000 7fecbdf45640 1 --2- 172.21.15.83:0/1844927704 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fecb8097170 0x7fecb8092890 unknown :-1 s=CLOSED pgs=37 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:04.636 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.634+0000 7fecbdf45640 1 --2- 172.21.15.83:0/1844927704 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fecb8096680 0x7fecb8092350 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:04.636 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.634+0000 7fecbdf45640 1 -- 172.21.15.83:0/1844927704 >> 172.21.15.83:0/1844927704 conn(0x7fecb809c6a0 msgr2=0x7fecb8128e70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:39:04.636 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.634+0000 7fecbdf45640 1 -- 172.21.15.83:0/1844927704 shutdown_connections 2024-09-14T03:39:04.636 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.634+0000 7fecbdf45640 1 -- 172.21.15.83:0/1844927704 wait complete. 2024-09-14T03:39:04.646 INFO:tasks.ceph.ceph_manager.ceph:all up! 2024-09-14T03:39:04.646 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-09-14T03:39:04.745 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.744+0000 7fef90432640 1 -- 172.21.15.83:0/1835473873 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fef88097170 msgr2=0x7fef88131060 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:04.746 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.744+0000 7fef90432640 1 --2- 172.21.15.83:0/1835473873 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fef88097170 0x7fef88131060 secure :-1 s=READY pgs=60 cs=0 l=1 rev1=1 crypto rx=0x7fef78009830 tx=0x7fef7802a480 comp rx=0 tx=0).stop 2024-09-14T03:39:04.746 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.744+0000 7fef90432640 1 -- 172.21.15.83:0/1835473873 shutdown_connections 2024-09-14T03:39:04.746 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.744+0000 7fef90432640 1 --2- 172.21.15.83:0/1835473873 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fef881315a0 0x7fef88133990 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:04.746 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.744+0000 7fef90432640 1 --2- 172.21.15.83:0/1835473873 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fef88097170 0x7fef88131060 unknown :-1 s=CLOSED pgs=60 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:04.746 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.744+0000 7fef90432640 1 -- 172.21.15.83:0/1835473873 >> 172.21.15.83:0/1835473873 conn(0x7fef8809c6a0 msgr2=0x7fef8809cab0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:39:04.746 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.744+0000 7fef90432640 1 -- 172.21.15.83:0/1835473873 shutdown_connections 2024-09-14T03:39:04.746 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.744+0000 7fef90432640 1 -- 172.21.15.83:0/1835473873 wait complete. 2024-09-14T03:39:04.748 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.745+0000 7fef90432640 1 Processor -- start 2024-09-14T03:39:04.749 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.745+0000 7fef90432640 1 -- start start 2024-09-14T03:39:04.749 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.745+0000 7fef90432640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fef88096680 0x7fef881234a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:39:04.749 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.745+0000 7fef90432640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fef88097170 0x7fef881239e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:39:04.749 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.745+0000 7fef90432640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fef881315a0 0x7fef88123f20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:39:04.749 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.745+0000 7fef90432640 1 -- --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7fef881347d0 con 0x7fef88096680 2024-09-14T03:39:04.749 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.745+0000 7fef90432640 1 -- --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7fef88134650 con 0x7fef88097170 2024-09-14T03:39:04.749 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.745+0000 7fef90432640 1 -- --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7fef88134950 con 0x7fef881315a0 2024-09-14T03:39:04.749 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.745+0000 7fef8e1ce640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fef88096680 0x7fef881234a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:39:04.749 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.745+0000 7fef8d9cd640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fef88097170 0x7fef881239e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:39:04.749 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.745+0000 7fef8d9cd640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fef88097170 0x7fef881239e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3301/0 says I am v2:172.21.15.83:60468/0 (socket says 172.21.15.83:60468) 2024-09-14T03:39:04.750 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.745+0000 7fef8e9cf640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fef881315a0 0x7fef88123f20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:39:04.750 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.745+0000 7fef8d9cd640 1 -- 172.21.15.83:0/4079130682 learned_addr learned my addr 172.21.15.83:0/4079130682 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:39:04.750 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.745+0000 7fef8e1ce640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fef88096680 0x7fef881234a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3300/0 says I am v2:172.21.15.83:35850/0 (socket says 172.21.15.83:35850) 2024-09-14T03:39:04.750 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.745+0000 7fef8d9cd640 1 -- 172.21.15.83:0/4079130682 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fef881315a0 msgr2=0x7fef88123f20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:04.750 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.745+0000 7fef8d9cd640 1 --2- 172.21.15.83:0/4079130682 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fef881315a0 0x7fef88123f20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:04.750 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.745+0000 7fef8d9cd640 1 -- 172.21.15.83:0/4079130682 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fef88096680 msgr2=0x7fef881234a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:04.750 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.745+0000 7fef8d9cd640 1 --2- 172.21.15.83:0/4079130682 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fef88096680 0x7fef881234a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:04.750 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.745+0000 7fef8d9cd640 1 -- 172.21.15.83:0/4079130682 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fef88124da0 con 0x7fef88097170 2024-09-14T03:39:04.750 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.746+0000 7fef8d9cd640 1 --2- 172.21.15.83:0/4079130682 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fef88097170 0x7fef881239e0 secure :-1 s=READY pgs=38 cs=0 l=1 rev1=1 crypto rx=0x7fef78009950 tx=0x7fef78032c10 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:39:04.750 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.746+0000 7fef7effd640 1 -- 172.21.15.83:0/4079130682 <== mon.1 v2:172.21.15.83:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fef78040070 con 0x7fef88097170 2024-09-14T03:39:04.750 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.746+0000 7fef7effd640 1 -- 172.21.15.83:0/4079130682 <== mon.1 v2:172.21.15.83:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fef78005310 con 0x7fef88097170 2024-09-14T03:39:04.750 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.746+0000 7fef90432640 1 -- 172.21.15.83:0/4079130682 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fef8806fb20 con 0x7fef88097170 2024-09-14T03:39:04.750 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.746+0000 7fef7effd640 1 -- 172.21.15.83:0/4079130682 <== mon.1 v2:172.21.15.83:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fef78005c90 con 0x7fef88097170 2024-09-14T03:39:04.750 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.746+0000 7fef90432640 1 -- 172.21.15.83:0/4079130682 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7fef881251c0 con 0x7fef88097170 2024-09-14T03:39:04.751 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.747+0000 7fef7effd640 1 -- 172.21.15.83:0/4079130682 <== mon.1 v2:172.21.15.83:3301/0 4 ==== mgrmap(e 5) ==== 55999+0+0 (secure 0 0 0) 0x7fef78039040 con 0x7fef88097170 2024-09-14T03:39:04.751 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.747+0000 7fef90432640 1 -- 172.21.15.83:0/4079130682 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fef88096a60 con 0x7fef88097170 2024-09-14T03:39:04.751 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.747+0000 7fef7effd640 1 -- 172.21.15.83:0/4079130682 <== mon.1 v2:172.21.15.83:3301/0 5 ==== osd_map(18..18 src has 1..18) ==== 2868+0+0 (secure 0 0 0) 0x7fef7807eb90 con 0x7fef88097170 2024-09-14T03:39:04.752 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.751+0000 7fef7effd640 1 -- 172.21.15.83:0/4079130682 <== mon.1 v2:172.21.15.83:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+151260 (secure 0 0 0) 0x7fef7807dce0 con 0x7fef88097170 2024-09-14T03:39:04.899 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.897+0000 7fef90432640 1 -- 172.21.15.83:0/4079130682 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) -- 0x7fef8812b040 con 0x7fef88097170 2024-09-14T03:39:04.900 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.898+0000 7fef7effd640 1 -- 172.21.15.83:0/4079130682 <== mon.1 v2:172.21.15.83:3301/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v18) ==== 74+0+8917 (secure 0 0 0) 0x7fef7807d820 con 0x7fef88097170 2024-09-14T03:39:04.900 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:39:04.901 INFO:teuthology.orchestra.run.smithi083.stdout:{"epoch":18,"fsid":"765298ef-df4c-4f28-af02-b45f60264ff9","created":"2024-09-14T03:38:43.954952+0000","modified":"2024-09-14T03:39:03.805107+0000","last_up_change":"2024-09-14T03:38:56.772236+0000","last_in_change":"2024-09-14T03:38:46.006167+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-14T03:38:56.922548+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"14","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":4,"score_stable":4,"optimal_score":0.5,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-14T03:39:00.433011+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":18,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.5,"score_stable":2.5,"optimal_score":1,"raw_score_acting":2.5,"raw_score_stable":2.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ff477614-343d-4d97-9df5-94933f9b904d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6805","nonce":2271402019}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6806","nonce":2271402019}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6808","nonce":2271402019}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6807","nonce":2271402019}]},"public_addr":"172.21.15.83:6805/2271402019","cluster_addr":"172.21.15.83:6806/2271402019","heartbeat_back_addr":"172.21.15.83:6808/2271402019","heartbeat_front_addr":"172.21.15.83:6807/2271402019","state":["exists","up"]},{"osd":1,"uuid":"562ec9a0-3f91-4a29-896d-94340e9fa36f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":11,"up_thru":15,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6801","nonce":2493296278}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6802","nonce":2493296278}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6804","nonce":2493296278}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6803","nonce":2493296278}]},"public_addr":"172.21.15.83:6801/2493296278","cluster_addr":"172.21.15.83:6802/2493296278","heartbeat_back_addr":"172.21.15.83:6804/2493296278","heartbeat_front_addr":"172.21.15.83:6803/2493296278","state":["exists","up"]},{"osd":2,"uuid":"8c242dda-3308-4804-b2be-9a1737c42cf6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":11,"up_thru":15,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6810","nonce":1553686396}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6812","nonce":1553686396}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6816","nonce":1553686396}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6814","nonce":1553686396}]},"public_addr":"172.21.15.83:6810/1553686396","cluster_addr":"172.21.15.83:6812/1553686396","heartbeat_back_addr":"172.21.15.83:6816/1553686396","heartbeat_front_addr":"172.21.15.83:6814/1553686396","state":["exists","up"]},{"osd":3,"uuid":"c4dd2c36-3c86-4b56-86fb-fede641a436c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":11,"up_thru":15,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6809","nonce":2350261690}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6811","nonce":2350261690}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6815","nonce":2350261690}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6813","nonce":2350261690}]},"public_addr":"172.21.15.83:6809/2350261690","cluster_addr":"172.21.15.83:6811/2350261690","heartbeat_back_addr":"172.21.15.83:6815/2350261690","heartbeat_front_addr":"172.21.15.83:6813/2350261690","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:38:53.047098+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:38:53.354351+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:38:53.422284+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:38:53.620657+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-14T03:39:04.902 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.900+0000 7fef90432640 1 -- 172.21.15.83:0/4079130682 >> v1:172.21.15.83:6800/1934165865 conn(0x7fef6c047fd0 legacy=0x7fef6c04a490 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:04.902 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.900+0000 7fef90432640 1 -- 172.21.15.83:0/4079130682 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fef88097170 msgr2=0x7fef881239e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:04.902 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.900+0000 7fef90432640 1 --2- 172.21.15.83:0/4079130682 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fef88097170 0x7fef881239e0 secure :-1 s=READY pgs=38 cs=0 l=1 rev1=1 crypto rx=0x7fef78009950 tx=0x7fef78032c10 comp rx=0 tx=0).stop 2024-09-14T03:39:04.902 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.900+0000 7fef90432640 1 -- 172.21.15.83:0/4079130682 shutdown_connections 2024-09-14T03:39:04.902 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.900+0000 7fef90432640 1 --2- 172.21.15.83:0/4079130682 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fef881315a0 0x7fef88123f20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:04.902 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.900+0000 7fef90432640 1 --2- 172.21.15.83:0/4079130682 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fef88097170 0x7fef881239e0 unknown :-1 s=CLOSED pgs=38 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:04.902 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.900+0000 7fef90432640 1 --2- 172.21.15.83:0/4079130682 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fef88096680 0x7fef881234a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:04.902 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.900+0000 7fef90432640 1 -- 172.21.15.83:0/4079130682 >> 172.21.15.83:0/4079130682 conn(0x7fef8809c6a0 msgr2=0x7fef88128e20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:39:04.902 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.900+0000 7fef90432640 1 -- 172.21.15.83:0/4079130682 shutdown_connections 2024-09-14T03:39:04.902 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:04.900+0000 7fef90432640 1 -- 172.21.15.83:0/4079130682 wait complete. 2024-09-14T03:39:04.912 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.0 flush_pg_stats 2024-09-14T03:39:04.913 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.1 flush_pg_stats 2024-09-14T03:39:04.913 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.2 flush_pg_stats 2024-09-14T03:39:04.914 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.3 flush_pg_stats 2024-09-14T03:39:05.021 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.019+0000 7f81e9fc3640 1 -- 172.21.15.83:0/4069891223 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f81e41302e0 msgr2=0x7f81e41326d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:05.021 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.019+0000 7f81e9fc3640 1 --2- 172.21.15.83:0/4069891223 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f81e41302e0 0x7f81e41326d0 secure :-1 s=READY pgs=38 cs=0 l=1 rev1=1 crypto rx=0x7f81dc009cd0 tx=0x7f81dc02cb40 comp rx=0 tx=0).stop 2024-09-14T03:39:05.022 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.019+0000 7f81e9fc3640 1 -- 172.21.15.83:0/4069891223 shutdown_connections 2024-09-14T03:39:05.022 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.019+0000 7f81e9fc3640 1 --2- 172.21.15.83:0/4069891223 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f81e41302e0 0x7f81e41326d0 unknown :-1 s=CLOSED pgs=38 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:05.022 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.019+0000 7f81e9fc3640 1 --2- 172.21.15.83:0/4069891223 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f81e409c730 0x7f81e4095670 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:05.022 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.019+0000 7f81e9fc3640 1 -- 172.21.15.83:0/4069891223 >> 172.21.15.83:0/4069891223 conn(0x7f81e409b660 msgr2=0x7f81e409ba70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:39:05.022 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.019+0000 7f81e9fc3640 1 -- 172.21.15.83:0/4069891223 shutdown_connections 2024-09-14T03:39:05.022 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.019+0000 7f81e9fc3640 1 -- 172.21.15.83:0/4069891223 wait complete. 2024-09-14T03:39:05.022 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.020+0000 7f2892d5e640 1 -- 172.21.15.83:0/363010740 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f288c1318a0 msgr2=0x7f288c133c90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:05.022 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.020+0000 7f2892d5e640 1 --2- 172.21.15.83:0/363010740 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f288c1318a0 0x7f288c133c90 secure :-1 s=READY pgs=40 cs=0 l=1 rev1=1 crypto rx=0x7f2884009be0 tx=0x7f288402a6f0 comp rx=0 tx=0).stop 2024-09-14T03:39:05.023 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.020+0000 7fa7ea5a5640 1 -- 172.21.15.83:0/237172624 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fa7e41334f0 msgr2=0x7fa7e41358e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:05.023 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.020+0000 7fa7ea5a5640 1 --2- 172.21.15.83:0/237172624 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fa7e41334f0 0x7fa7e41358e0 secure :-1 s=READY pgs=39 cs=0 l=1 rev1=1 crypto rx=0x7fa7dc009c10 tx=0x7fa7dc02a780 comp rx=0 tx=0).stop 2024-09-14T03:39:05.023 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.020+0000 7f7393848640 1 -- 172.21.15.83:0/1729691912 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f738c091bc0 msgr2=0x7f738c091fa0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:05.023 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.020+0000 7f7393848640 1 --2- 172.21.15.83:0/1729691912 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f738c091bc0 0x7f738c091fa0 secure :-1 s=READY pgs=39 cs=0 l=1 rev1=1 crypto rx=0x7f737c009830 tx=0x7f737c02a3f0 comp rx=0 tx=0).stop 2024-09-14T03:39:05.024 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.020+0000 7f2892d5e640 1 -- 172.21.15.83:0/363010740 shutdown_connections 2024-09-14T03:39:05.024 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.020+0000 7f2892d5e640 1 --2- 172.21.15.83:0/363010740 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f288c1318a0 0x7f288c133c90 unknown :-1 s=CLOSED pgs=40 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:05.024 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.020+0000 7f2892d5e640 1 --2- 172.21.15.83:0/363010740 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f288c0971e0 0x7f288c131360 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:05.024 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.020+0000 7f2892d5e640 1 -- 172.21.15.83:0/363010740 >> 172.21.15.83:0/363010740 conn(0x7f288c09c710 msgr2=0x7f288c09cb20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:39:05.024 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.020+0000 7f7393848640 1 -- 172.21.15.83:0/1729691912 shutdown_connections 2024-09-14T03:39:05.024 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.020+0000 7f7393848640 1 --2- 172.21.15.83:0/1729691912 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f738c091bc0 0x7f738c091fa0 unknown :-1 s=CLOSED pgs=39 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:05.025 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.020+0000 7fa7ea5a5640 1 -- 172.21.15.83:0/237172624 shutdown_connections 2024-09-14T03:39:05.025 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.020+0000 7fa7ea5a5640 1 --2- 172.21.15.83:0/237172624 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fa7e41334f0 0x7fa7e41358e0 unknown :-1 s=CLOSED pgs=39 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:05.025 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.020+0000 7fa7ea5a5640 1 --2- 172.21.15.83:0/237172624 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fa7e408d270 0x7fa7e4132fb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:05.025 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.020+0000 7fa7ea5a5640 1 -- 172.21.15.83:0/237172624 >> 172.21.15.83:0/237172624 conn(0x7fa7e40a0c80 msgr2=0x7fa7e408b000 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:39:05.025 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.020+0000 7f7393848640 1 --2- 172.21.15.83:0/1729691912 >> v2:172.21.15.83:3300/0 conn(0x7f738c097280 0x7f738c091680 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:05.025 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.020+0000 7f7393848640 1 -- 172.21.15.83:0/1729691912 >> 172.21.15.83:0/1729691912 conn(0x7f738c09d510 msgr2=0x7f738c09d920 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:39:05.026 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.020+0000 7fa7ea5a5640 1 -- 172.21.15.83:0/237172624 shutdown_connections 2024-09-14T03:39:05.026 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.020+0000 7fa7ea5a5640 1 -- 172.21.15.83:0/237172624 wait complete. 2024-09-14T03:39:05.026 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.020+0000 7f2892d5e640 1 -- 172.21.15.83:0/363010740 shutdown_connections 2024-09-14T03:39:05.026 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.020+0000 7f2892d5e640 1 -- 172.21.15.83:0/363010740 wait complete. 2024-09-14T03:39:05.026 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.020+0000 7f7393848640 1 -- 172.21.15.83:0/1729691912 shutdown_connections 2024-09-14T03:39:05.027 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.020+0000 7f7393848640 1 -- 172.21.15.83:0/1729691912 wait complete. 2024-09-14T03:39:05.027 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.021+0000 7f81e9fc3640 1 Processor -- start 2024-09-14T03:39:05.027 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.021+0000 7fa7ea5a5640 1 Processor -- start 2024-09-14T03:39:05.027 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.021+0000 7fa7ea5a5640 1 -- start start 2024-09-14T03:39:05.028 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.021+0000 7f81e9fc3640 1 -- start start 2024-09-14T03:39:05.028 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.021+0000 7f2892d5e640 1 Processor -- start 2024-09-14T03:39:05.028 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.021+0000 7f2892d5e640 1 -- start start 2024-09-14T03:39:05.028 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.021+0000 7fa7ea5a5640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fa7e408c780 0x7fa7e409a260 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:39:05.029 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.021+0000 7f81e9fc3640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f81e4096720 0x7f81e412f560 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:39:05.029 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.021+0000 7f81e9fc3640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f81e409c730 0x7f81e412faa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:39:05.029 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.021+0000 7f81e9fc3640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f81e41302e0 0x7f81e408bb40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:39:05.029 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.021+0000 7f81e9fc3640 1 -- --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7f81e406c710 con 0x7f81e41302e0 2024-09-14T03:39:05.029 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.021+0000 7f81e9fc3640 1 -- --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7f81e406c590 con 0x7f81e409c730 2024-09-14T03:39:05.029 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.021+0000 7fa7ea5a5640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fa7e408d270 0x7fa7e409a7a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:39:05.030 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.021+0000 7f81e9fc3640 1 -- --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7f81e406c890 con 0x7f81e4096720 2024-09-14T03:39:05.030 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.021+0000 7f7393848640 1 Processor -- start 2024-09-14T03:39:05.031 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.021+0000 7fa7e3fff640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fa7e408c780 0x7fa7e409a260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:39:05.031 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.021+0000 7fa7e3fff640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fa7e408c780 0x7fa7e409a260 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3302/0 says I am v2:172.21.15.83:32806/0 (socket says 172.21.15.83:32806) 2024-09-14T03:39:05.031 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.021+0000 7fa7ea5a5640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fa7e41334f0 0x7fa7e409ace0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:39:05.031 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.021+0000 7fa7e3fff640 1 -- 172.21.15.83:0/2615111424 learned_addr learned my addr 172.21.15.83:0/2615111424 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:39:05.031 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.022+0000 7fa7ea5a5640 1 -- 172.21.15.83:0/2615111424 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7fa7e41369e0 con 0x7fa7e41334f0 2024-09-14T03:39:05.031 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.022+0000 7fa7ea5a5640 1 -- 172.21.15.83:0/2615111424 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7fa7e4136860 con 0x7fa7e408d270 2024-09-14T03:39:05.031 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.022+0000 7fa7ea5a5640 1 -- 172.21.15.83:0/2615111424 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7fa7e4136b60 con 0x7fa7e408c780 2024-09-14T03:39:05.031 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.022+0000 7f7393848640 1 -- start start 2024-09-14T03:39:05.032 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.022+0000 7f81e2ffd640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f81e409c730 0x7f81e412faa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:39:05.032 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.022+0000 7fa7e37fe640 1 --2- 172.21.15.83:0/2615111424 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fa7e408d270 0x7fa7e409a7a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:39:05.032 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.022+0000 7fa7e8b42640 1 --2- 172.21.15.83:0/2615111424 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fa7e41334f0 0x7fa7e409ace0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:39:05.032 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.022+0000 7f81e3fff640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f81e41302e0 0x7f81e408bb40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:39:05.033 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.022+0000 7fa7e3fff640 1 -- 172.21.15.83:0/2615111424 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fa7e408d270 msgr2=0x7fa7e409a7a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:05.033 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.022+0000 7f81e2ffd640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f81e409c730 0x7f81e412faa0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3301/0 says I am v2:172.21.15.83:60520/0 (socket says 172.21.15.83:60520) 2024-09-14T03:39:05.033 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.022+0000 7f81e2ffd640 1 -- 172.21.15.83:0/3407706212 learned_addr learned my addr 172.21.15.83:0/3407706212 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:39:05.033 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.022+0000 7f81e37fe640 1 --2- 172.21.15.83:0/3407706212 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f81e4096720 0x7f81e412f560 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:39:05.034 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.022+0000 7fa7e3fff640 1 --2- 172.21.15.83:0/2615111424 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fa7e408d270 0x7fa7e409a7a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:05.034 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.022+0000 7fa7e3fff640 1 -- 172.21.15.83:0/2615111424 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fa7e41334f0 msgr2=0x7fa7e409ace0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:05.034 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.022+0000 7fa7e3fff640 1 --2- 172.21.15.83:0/2615111424 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fa7e41334f0 0x7fa7e409ace0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:05.034 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.022+0000 7fa7e3fff640 1 -- 172.21.15.83:0/2615111424 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fa7e409cba0 con 0x7fa7e408c780 2024-09-14T03:39:05.034 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.022+0000 7fa7e37fe640 1 --2- 172.21.15.83:0/2615111424 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fa7e408d270 0x7fa7e409a7a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:39:05.034 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.022+0000 7f2892d5e640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f288c0966f0 0x7f288c1305e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:39:05.034 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.022+0000 7f2892d5e640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f288c0971e0 0x7f288c130b20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:39:05.034 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.022+0000 7f2892d5e640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f288c1318a0 0x7f288c08dff0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:39:05.035 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.022+0000 7f2892d5e640 1 -- --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7f288c134ae0 con 0x7f288c0971e0 2024-09-14T03:39:05.035 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.022+0000 7f2892d5e640 1 -- --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7f288c134960 con 0x7f288c0966f0 2024-09-14T03:39:05.035 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.022+0000 7f2892d5e640 1 -- --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7f288c134c60 con 0x7f288c1318a0 2024-09-14T03:39:05.035 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.022+0000 7f2890afa640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f288c0966f0 0x7f288c1305e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:39:05.035 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.022+0000 7f2890afa640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f288c0966f0 0x7f288c1305e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3301/0 says I am v2:172.21.15.83:60538/0 (socket says 172.21.15.83:60538) 2024-09-14T03:39:05.035 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.022+0000 7f2890afa640 1 -- 172.21.15.83:0/3440063112 learned_addr learned my addr 172.21.15.83:0/3440063112 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:39:05.035 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.022+0000 7f28912fb640 1 --2- 172.21.15.83:0/3440063112 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f288c1318a0 0x7f288c08dff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:39:05.035 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.022+0000 7f288bfff640 1 --2- 172.21.15.83:0/3440063112 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f288c0971e0 0x7f288c130b20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:39:05.035 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.022+0000 7fa7e3fff640 1 --2- 172.21.15.83:0/2615111424 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fa7e408c780 0x7fa7e409a260 secure :-1 s=READY pgs=41 cs=0 l=1 rev1=1 crypto rx=0x7fa7d400c3c0 tx=0x7fa7d400c880 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:39:05.036 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.022+0000 7f81e3fff640 1 -- 172.21.15.83:0/3407706212 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f81e4096720 msgr2=0x7f81e412f560 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:05.036 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.022+0000 7f81e3fff640 1 --2- 172.21.15.83:0/3407706212 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f81e4096720 0x7f81e412f560 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:05.036 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.022+0000 7f81e3fff640 1 -- 172.21.15.83:0/3407706212 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f81e409c730 msgr2=0x7f81e412faa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:05.036 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.022+0000 7f81e3fff640 1 --2- 172.21.15.83:0/3407706212 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f81e409c730 0x7f81e412faa0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:05.036 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.022+0000 7f81e3fff640 1 -- 172.21.15.83:0/3407706212 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f81e412ffe0 con 0x7f81e41302e0 2024-09-14T03:39:05.036 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.023+0000 7fa7e0ff9640 1 -- 172.21.15.83:0/2615111424 <== mon.2 v2:172.21.15.83:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa7d4017070 con 0x7fa7e408c780 2024-09-14T03:39:05.037 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.023+0000 7f28912fb640 1 -- 172.21.15.83:0/3440063112 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f288c0966f0 msgr2=0x7f288c1305e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:05.037 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.023+0000 7f28912fb640 1 --2- 172.21.15.83:0/3440063112 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f288c0966f0 0x7f288c1305e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:05.037 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.023+0000 7f28912fb640 1 -- 172.21.15.83:0/3440063112 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f288c0971e0 msgr2=0x7f288c130b20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:05.037 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.023+0000 7f28912fb640 1 --2- 172.21.15.83:0/3440063112 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f288c0971e0 0x7f288c130b20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:05.037 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.023+0000 7f28912fb640 1 -- 172.21.15.83:0/3440063112 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f288c0917f0 con 0x7f288c1318a0 2024-09-14T03:39:05.037 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.022+0000 7f7393848640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f738c091bc0 0x7f738c12e270 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:39:05.038 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.022+0000 7f7393848640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f738c097280 0x7f738c12e7b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:39:05.038 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.023+0000 7f7393848640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f738c12f260 0x7f738c12ecf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:39:05.038 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.023+0000 7f73915e4640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f738c091bc0 0x7f738c12e270 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:39:05.038 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.023+0000 7f73915e4640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f738c091bc0 0x7f738c12e270 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3301/0 says I am v2:172.21.15.83:60546/0 (socket says 172.21.15.83:60546) 2024-09-14T03:39:05.038 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.023+0000 7f73915e4640 1 -- 172.21.15.83:0/2309084977 learned_addr learned my addr 172.21.15.83:0/2309084977 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:39:05.038 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.023+0000 7f7393848640 1 -- 172.21.15.83:0/2309084977 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7f738c06c660 con 0x7f738c12f260 2024-09-14T03:39:05.038 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.023+0000 7f7393848640 1 -- 172.21.15.83:0/2309084977 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7f738c06c4e0 con 0x7f738c091bc0 2024-09-14T03:39:05.038 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.023+0000 7f7393848640 1 -- 172.21.15.83:0/2309084977 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7f738c06c7e0 con 0x7f738c097280 2024-09-14T03:39:05.038 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.023+0000 7fa7e0ff9640 1 -- 172.21.15.83:0/2615111424 <== mon.2 v2:172.21.15.83:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fa7d40035f0 con 0x7fa7e408c780 2024-09-14T03:39:05.038 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.023+0000 7fa7e0ff9640 1 -- 172.21.15.83:0/2615111424 <== mon.2 v2:172.21.15.83:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa7d4003a40 con 0x7fa7e408c780 2024-09-14T03:39:05.038 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.023+0000 7fa7ea5a5640 1 -- 172.21.15.83:0/2615111424 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fa7e409ceb0 con 0x7fa7e408c780 2024-09-14T03:39:05.038 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.023+0000 7fa7ea5a5640 1 -- 172.21.15.83:0/2615111424 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7fa7e409bb80 con 0x7fa7e408c780 2024-09-14T03:39:05.039 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.023+0000 7f28912fb640 1 --2- 172.21.15.83:0/3440063112 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f288c1318a0 0x7f288c08dff0 secure :-1 s=READY pgs=42 cs=0 l=1 rev1=1 crypto rx=0x7f2884009000 tx=0x7f2884032ce0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:39:05.039 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.023+0000 7f7390de3640 1 --2- 172.21.15.83:0/2309084977 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f738c097280 0x7f738c12e7b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:39:05.039 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.023+0000 7f73915e4640 1 -- 172.21.15.83:0/2309084977 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f738c097280 msgr2=0x7f738c12e7b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:05.039 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.023+0000 7f73915e4640 1 --2- 172.21.15.83:0/2309084977 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f738c097280 0x7f738c12e7b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:05.039 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.023+0000 7f73915e4640 1 -- 172.21.15.83:0/2309084977 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f738c12f260 msgr2=0x7f738c12ecf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:05.040 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.023+0000 7f81e3fff640 1 --2- 172.21.15.83:0/3407706212 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f81e41302e0 0x7f81e408bb40 secure :-1 s=READY pgs=64 cs=0 l=1 rev1=1 crypto rx=0x7f81dc034000 tx=0x7f81dc003c10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:39:05.040 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.023+0000 7f73915e4640 1 --2- 172.21.15.83:0/2309084977 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f738c12f260 0x7f738c12ecf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:05.040 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.023+0000 7f73915e4640 1 -- 172.21.15.83:0/2309084977 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f738c08d270 con 0x7f738c091bc0 2024-09-14T03:39:05.041 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.023+0000 7f81cbfff640 1 -- 172.21.15.83:0/3407706212 <== mon.0 v2:172.21.15.83:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f81dc046070 con 0x7f81e41302e0 2024-09-14T03:39:05.041 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.023+0000 7f81cbfff640 1 -- 172.21.15.83:0/3407706212 <== mon.0 v2:172.21.15.83:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f81dc004250 con 0x7f81e41302e0 2024-09-14T03:39:05.041 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.023+0000 7f81cbfff640 1 -- 172.21.15.83:0/3407706212 <== mon.0 v2:172.21.15.83:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f81dc0046a0 con 0x7f81e41302e0 2024-09-14T03:39:05.041 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.023+0000 7f81e9fc3640 1 -- 172.21.15.83:0/3407706212 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f81e408e710 con 0x7f81e41302e0 2024-09-14T03:39:05.041 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.023+0000 7f73915e4640 1 --2- 172.21.15.83:0/2309084977 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f738c091bc0 0x7f738c12e270 secure :-1 s=READY pgs=40 cs=0 l=1 rev1=1 crypto rx=0x7f73880099c0 tx=0x7f7388009e80 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:39:05.041 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.024+0000 7f7381ffb640 1 -- 172.21.15.83:0/2309084977 <== mon.1 v2:172.21.15.83:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f738801b070 con 0x7f738c091bc0 2024-09-14T03:39:05.042 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.024+0000 7f7393848640 1 -- 172.21.15.83:0/2309084977 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f738c08e470 con 0x7f738c091bc0 2024-09-14T03:39:05.042 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.023+0000 7f81e9fc3640 1 -- 172.21.15.83:0/3407706212 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f81e408c080 con 0x7f81e41302e0 2024-09-14T03:39:05.043 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.024+0000 7f7393848640 1 -- 172.21.15.83:0/2309084977 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7f738c08b8e0 con 0x7f738c091bc0 2024-09-14T03:39:05.043 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.024+0000 7f7381ffb640 1 -- 172.21.15.83:0/2309084977 <== mon.1 v2:172.21.15.83:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f738800ca60 con 0x7f738c091bc0 2024-09-14T03:39:05.043 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.024+0000 7f28897fa640 1 -- 172.21.15.83:0/3440063112 <== mon.2 v2:172.21.15.83:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2884041020 con 0x7f288c1318a0 2024-09-14T03:39:05.043 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.024+0000 7f28897fa640 1 -- 172.21.15.83:0/3440063112 <== mon.2 v2:172.21.15.83:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f2884004040 con 0x7f288c1318a0 2024-09-14T03:39:05.044 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.024+0000 7f28897fa640 1 -- 172.21.15.83:0/3440063112 <== mon.2 v2:172.21.15.83:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2884036040 con 0x7f288c1318a0 2024-09-14T03:39:05.044 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.024+0000 7f81e9fc3640 1 -- 172.21.15.83:0/3407706212 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_get_version(what=osdmap handle=1) -- 0x7f81e408dff0 con 0x7f81e41302e0 2024-09-14T03:39:05.045 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.024+0000 7fa7ea5a5640 1 -- 172.21.15.83:0/2615111424 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_get_version(what=osdmap handle=1) -- 0x7fa7e412e230 con 0x7fa7e408c780 2024-09-14T03:39:05.045 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.024+0000 7f2892d5e640 1 -- 172.21.15.83:0/3440063112 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f288c090240 con 0x7f288c1318a0 2024-09-14T03:39:05.046 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.024+0000 7f2892d5e640 1 -- 172.21.15.83:0/3440063112 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7f288c090c40 con 0x7f288c1318a0 2024-09-14T03:39:05.046 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.025+0000 7fa7e0ff9640 1 -- 172.21.15.83:0/2615111424 <== mon.2 v2:172.21.15.83:3302/0 4 ==== mgrmap(e 5) ==== 55999+0+0 (secure 0 0 0) 0x7fa7d401a070 con 0x7fa7e408c780 2024-09-14T03:39:05.047 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.025+0000 7f7381ffb640 1 -- 172.21.15.83:0/2309084977 <== mon.1 v2:172.21.15.83:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f738800ce90 con 0x7f738c091bc0 2024-09-14T03:39:05.047 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.025+0000 7f7393848640 1 -- 172.21.15.83:0/2309084977 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_get_version(what=osdmap handle=1) -- 0x7f736c001030 con 0x7f738c091bc0 2024-09-14T03:39:05.048 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.025+0000 7fa7e0ff9640 1 -- 172.21.15.83:0/2615111424 <== mon.2 v2:172.21.15.83:3302/0 5 ==== osd_map(18..18 src has 1..18) ==== 2868+0+0 (secure 0 0 0) 0x7fa7d405cba0 con 0x7fa7e408c780 2024-09-14T03:39:05.048 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.025+0000 7fa7e0ff9640 1 -- 172.21.15.83:0/2615111424 --> v1:172.21.15.83:6805/2271402019 -- command(tid 1: {"prefix": "get_command_descriptions"}) -- 0x7fa7d0052430 con 0x7fa7d004e740 2024-09-14T03:39:05.048 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.025+0000 7f81cbfff640 1 -- 172.21.15.83:0/3407706212 <== mon.0 v2:172.21.15.83:3300/0 4 ==== mgrmap(e 5) ==== 55999+0+0 (secure 0 0 0) 0x7f81dc02f430 con 0x7f81e41302e0 2024-09-14T03:39:05.049 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.025+0000 7fa7e0ff9640 1 -- 172.21.15.83:0/2615111424 <== mon.2 v2:172.21.15.83:3302/0 6 ==== mon_get_version_reply(handle=1 version=18) ==== 24+0+0 (secure 0 0 0) 0x7fa7d403d2c0 con 0x7fa7e408c780 2024-09-14T03:39:05.049 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.026+0000 7f28897fa640 1 -- 172.21.15.83:0/3440063112 <== mon.2 v2:172.21.15.83:3302/0 4 ==== mgrmap(e 5) ==== 55999+0+0 (secure 0 0 0) 0x7f2884044020 con 0x7f288c1318a0 2024-09-14T03:39:05.050 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.026+0000 7f7381ffb640 1 -- 172.21.15.83:0/2309084977 <== mon.1 v2:172.21.15.83:3301/0 4 ==== mgrmap(e 5) ==== 55999+0+0 (secure 0 0 0) 0x7f738801e070 con 0x7f738c091bc0 2024-09-14T03:39:05.050 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.026+0000 7f2892d5e640 1 -- 172.21.15.83:0/3440063112 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_get_version(what=osdmap handle=1) -- 0x7f288c096ad0 con 0x7f288c1318a0 2024-09-14T03:39:05.051 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.026+0000 7f81cbfff640 1 -- 172.21.15.83:0/3407706212 <== mon.0 v2:172.21.15.83:3300/0 5 ==== osd_map(18..18 src has 1..18) ==== 2868+0+0 (secure 0 0 0) 0x7f81dc080f30 con 0x7f81e41302e0 2024-09-14T03:39:05.051 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.026+0000 7f81cbfff640 1 -- 172.21.15.83:0/3407706212 --> v1:172.21.15.83:6809/2350261690 -- command(tid 1: {"prefix": "get_command_descriptions"}) -- 0x7f81dc03be50 con 0x7f81d004e6a0 2024-09-14T03:39:05.051 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.026+0000 7f81cbfff640 1 -- 172.21.15.83:0/3407706212 <== mon.0 v2:172.21.15.83:3300/0 6 ==== mon_get_version_reply(handle=1 version=18) ==== 24+0+0 (secure 0 0 0) 0x7f81dc081410 con 0x7f81e41302e0 2024-09-14T03:39:05.051 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.026+0000 7f28897fa640 1 -- 172.21.15.83:0/3440063112 <== mon.2 v2:172.21.15.83:3302/0 5 ==== osd_map(18..18 src has 1..18) ==== 2868+0+0 (secure 0 0 0) 0x7f288407e250 con 0x7f288c1318a0 2024-09-14T03:39:05.052 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.026+0000 7f7381ffb640 1 -- 172.21.15.83:0/2309084977 <== mon.1 v2:172.21.15.83:3301/0 5 ==== osd_map(18..18 src has 1..18) ==== 2868+0+0 (secure 0 0 0) 0x7f738805d330 con 0x7f738c091bc0 2024-09-14T03:39:05.052 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.026+0000 7f28897fa640 1 -- 172.21.15.83:0/3440063112 --> v1:172.21.15.83:6801/2493296278 -- command(tid 1: {"prefix": "get_command_descriptions"}) -- 0x7f288402eec0 con 0x7f287004e790 2024-09-14T03:39:05.052 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.027+0000 7f28897fa640 1 -- 172.21.15.83:0/3440063112 <== mon.2 v2:172.21.15.83:3302/0 6 ==== mon_get_version_reply(handle=1 version=18) ==== 24+0+0 (secure 0 0 0) 0x7f2884085050 con 0x7f288c1318a0 2024-09-14T03:39:05.052 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.027+0000 7f7381ffb640 1 -- 172.21.15.83:0/2309084977 --> v1:172.21.15.83:6810/1553686396 -- command(tid 1: {"prefix": "get_command_descriptions"}) -- 0x7f738800ce90 con 0x7f737004e820 2024-09-14T03:39:05.053 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.027+0000 7f7381ffb640 1 -- 172.21.15.83:0/2309084977 <== mon.1 v2:172.21.15.83:3301/0 6 ==== mon_get_version_reply(handle=1 version=18) ==== 24+0+0 (secure 0 0 0) 0x7f7388064050 con 0x7f738c091bc0 2024-09-14T03:39:05.053 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.027+0000 7fa7e0ff9640 1 -- 172.21.15.83:0/2615111424 <== osd.0 v1:172.21.15.83:6805/2271402019 1 ==== command_reply(tid 1: 0 ) ==== 8+0+27193 (unknown 0 0 1302949928) 0x7fa7d0052430 con 0x7fa7d004e740 2024-09-14T03:39:05.053 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.028+0000 7f81cbfff640 1 -- 172.21.15.83:0/3407706212 <== osd.3 v1:172.21.15.83:6809/2350261690 1 ==== command_reply(tid 1: 0 ) ==== 8+0+27193 (unknown 0 0 1302949928) 0x7f81dc03be50 con 0x7f81d004e6a0 2024-09-14T03:39:05.053 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.028+0000 7f28897fa640 1 -- 172.21.15.83:0/3440063112 <== osd.1 v1:172.21.15.83:6801/2493296278 1 ==== command_reply(tid 1: 0 ) ==== 8+0+27193 (unknown 0 0 1302949928) 0x7f288402eec0 con 0x7f287004e790 2024-09-14T03:39:05.054 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.028+0000 7f7381ffb640 1 -- 172.21.15.83:0/2309084977 <== osd.2 v1:172.21.15.83:6810/1553686396 1 ==== command_reply(tid 1: 0 ) ==== 8+0+27193 (unknown 0 0 1302949928) 0x7f738800ce90 con 0x7f737004e820 2024-09-14T03:39:05.054 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.041+0000 7f2892d5e640 1 -- 172.21.15.83:0/3440063112 --> v1:172.21.15.83:6801/2493296278 -- command(tid 2: {"prefix": "flush_pg_stats"}) -- 0x7f288c092c30 con 0x7f287004e790 2024-09-14T03:39:05.054 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.041+0000 7f81e9fc3640 1 -- 172.21.15.83:0/3407706212 --> v1:172.21.15.83:6809/2350261690 -- command(tid 2: {"prefix": "flush_pg_stats"}) -- 0x7f81e408c5c0 con 0x7f81d004e6a0 2024-09-14T03:39:05.054 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.042+0000 7f81cbfff640 1 -- 172.21.15.83:0/3407706212 <== osd.3 v1:172.21.15.83:6809/2350261690 2 ==== command_reply(tid 2: 0 ) ==== 8+0+11 (unknown 0 0 654699149) 0x7f81e408c5c0 con 0x7f81d004e6a0 2024-09-14T03:39:05.055 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.042+0000 7f28897fa640 1 -- 172.21.15.83:0/3440063112 <== osd.1 v1:172.21.15.83:6801/2493296278 2 ==== command_reply(tid 2: 0 ) ==== 8+0+11 (unknown 0 0 654699149) 0x7f288c092c30 con 0x7f287004e790 2024-09-14T03:39:05.055 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.042+0000 7f2892d5e640 1 -- 172.21.15.83:0/3440063112 >> v1:172.21.15.83:6801/2493296278 conn(0x7f287004e790 legacy=0x7f2870050c10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:05.055 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.042+0000 7f81e9fc3640 1 -- 172.21.15.83:0/3407706212 >> v1:172.21.15.83:6809/2350261690 conn(0x7f81d004e6a0 legacy=0x7f81d0050b20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:05.056 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.042+0000 7f2892d5e640 1 -- 172.21.15.83:0/3440063112 >> v1:172.21.15.83:6800/1934165865 conn(0x7f28700480a0 legacy=0x7f287004a560 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:05.056 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.042+0000 7f2892d5e640 1 -- 172.21.15.83:0/3440063112 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f288c1318a0 msgr2=0x7f288c08dff0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:05.056 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.042+0000 7f2892d5e640 1 --2- 172.21.15.83:0/3440063112 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f288c1318a0 0x7f288c08dff0 secure :-1 s=READY pgs=42 cs=0 l=1 rev1=1 crypto rx=0x7f2884009000 tx=0x7f2884032ce0 comp rx=0 tx=0).stop 2024-09-14T03:39:05.057 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.042+0000 7f81e9fc3640 1 -- 172.21.15.83:0/3407706212 >> v1:172.21.15.83:6800/1934165865 conn(0x7f81d0048020 legacy=0x7f81d004a4e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:05.057 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.042+0000 7f28912fb640 1 -- 172.21.15.83:0/3440063112 reap_dead start 2024-09-14T03:39:05.057 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.042+0000 7f81e9fc3640 1 -- 172.21.15.83:0/3407706212 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f81e41302e0 msgr2=0x7f81e408bb40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:05.057 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.042+0000 7f81e9fc3640 1 --2- 172.21.15.83:0/3407706212 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f81e41302e0 0x7f81e408bb40 secure :-1 s=READY pgs=64 cs=0 l=1 rev1=1 crypto rx=0x7f81dc034000 tx=0x7f81dc003c10 comp rx=0 tx=0).stop 2024-09-14T03:39:05.057 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.042+0000 7f81e3fff640 1 -- 172.21.15.83:0/3407706212 reap_dead start 2024-09-14T03:39:05.058 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.043+0000 7f81e9fc3640 1 -- 172.21.15.83:0/3407706212 shutdown_connections 2024-09-14T03:39:05.058 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.043+0000 7f81e9fc3640 1 -- 172.21.15.83:0/3407706212 >> 172.21.15.83:0/3407706212 conn(0x7f81e409b660 msgr2=0x7f81e4098790 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:39:05.058 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.043+0000 7f2892d5e640 1 -- 172.21.15.83:0/3440063112 shutdown_connections 2024-09-14T03:39:05.058 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.043+0000 7f2892d5e640 1 -- 172.21.15.83:0/3440063112 >> 172.21.15.83:0/3440063112 conn(0x7f288c09c710 msgr2=0x7f288c06f8c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:39:05.058 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.043+0000 7f81e9fc3640 1 -- 172.21.15.83:0/3407706212 shutdown_connections 2024-09-14T03:39:05.058 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.043+0000 7f81e9fc3640 1 -- 172.21.15.83:0/3407706212 wait complete. 2024-09-14T03:39:05.059 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.043+0000 7f2892d5e640 1 -- 172.21.15.83:0/3440063112 shutdown_connections 2024-09-14T03:39:05.059 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.043+0000 7f2892d5e640 1 -- 172.21.15.83:0/3440063112 wait complete. 2024-09-14T03:39:05.059 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.048+0000 7fa7ea5a5640 1 -- 172.21.15.83:0/2615111424 --> v1:172.21.15.83:6805/2271402019 -- command(tid 2: {"prefix": "flush_pg_stats"}) -- 0x7fa7e4127e30 con 0x7fa7d004e740 2024-09-14T03:39:05.059 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.049+0000 7fa7e0ff9640 1 -- 172.21.15.83:0/2615111424 <== osd.0 v1:172.21.15.83:6805/2271402019 2 ==== command_reply(tid 2: 0 ) ==== 8+0+11 (unknown 0 0 3659784045) 0x7fa7e4127e30 con 0x7fa7d004e740 2024-09-14T03:39:05.060 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.049+0000 7fa7ea5a5640 1 -- 172.21.15.83:0/2615111424 >> v1:172.21.15.83:6805/2271402019 conn(0x7fa7d004e740 legacy=0x7fa7d0050bc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:05.060 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.049+0000 7fa7ea5a5640 1 -- 172.21.15.83:0/2615111424 >> v1:172.21.15.83:6800/1934165865 conn(0x7fa7d0048050 legacy=0x7fa7d004a510 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:05.061 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.049+0000 7fa7ea5a5640 1 -- 172.21.15.83:0/2615111424 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fa7e408c780 msgr2=0x7fa7e409a260 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:05.061 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.049+0000 7fa7ea5a5640 1 --2- 172.21.15.83:0/2615111424 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fa7e408c780 0x7fa7e409a260 secure :-1 s=READY pgs=41 cs=0 l=1 rev1=1 crypto rx=0x7fa7d400c3c0 tx=0x7fa7d400c880 comp rx=0 tx=0).stop 2024-09-14T03:39:05.061 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.049+0000 7fa7e8b42640 1 -- 172.21.15.83:0/2615111424 reap_dead start 2024-09-14T03:39:05.061 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.049+0000 7fa7ea5a5640 1 -- 172.21.15.83:0/2615111424 shutdown_connections 2024-09-14T03:39:05.061 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.049+0000 7fa7ea5a5640 1 -- 172.21.15.83:0/2615111424 >> 172.21.15.83:0/2615111424 conn(0x7fa7e40a0c80 msgr2=0x7fa7e4127720 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:39:05.061 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.050+0000 7fa7ea5a5640 1 -- 172.21.15.83:0/2615111424 shutdown_connections 2024-09-14T03:39:05.061 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.050+0000 7fa7ea5a5640 1 -- 172.21.15.83:0/2615111424 wait complete. 2024-09-14T03:39:05.061 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.050+0000 7f7393848640 1 -- 172.21.15.83:0/2309084977 --> v1:172.21.15.83:6810/1553686396 -- command(tid 2: {"prefix": "flush_pg_stats"}) -- 0x7f736c002dd0 con 0x7f737004e820 2024-09-14T03:39:05.061 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.050+0000 7f7381ffb640 1 -- 172.21.15.83:0/2309084977 <== osd.2 v1:172.21.15.83:6810/1553686396 2 ==== command_reply(tid 2: 0 ) ==== 8+0+11 (unknown 0 0 654699149) 0x7f736c002dd0 con 0x7f737004e820 2024-09-14T03:39:05.062 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.050+0000 7f7393848640 1 -- 172.21.15.83:0/2309084977 >> v1:172.21.15.83:6810/1553686396 conn(0x7f737004e820 legacy=0x7f7370050ca0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:05.062 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.051+0000 7f7393848640 1 -- 172.21.15.83:0/2309084977 >> v1:172.21.15.83:6800/1934165865 conn(0x7f73700481a0 legacy=0x7f737004a660 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:05.062 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.051+0000 7f7393848640 1 -- 172.21.15.83:0/2309084977 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f738c091bc0 msgr2=0x7f738c12e270 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:05.063 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.051+0000 7f7393848640 1 --2- 172.21.15.83:0/2309084977 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f738c091bc0 0x7f738c12e270 secure :-1 s=READY pgs=40 cs=0 l=1 rev1=1 crypto rx=0x7f73880099c0 tx=0x7f7388009e80 comp rx=0 tx=0).stop 2024-09-14T03:39:05.063 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.051+0000 7f7391de5640 1 -- 172.21.15.83:0/2309084977 reap_dead start 2024-09-14T03:39:05.063 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.051+0000 7f7393848640 1 -- 172.21.15.83:0/2309084977 shutdown_connections 2024-09-14T03:39:05.063 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.051+0000 7f7393848640 1 -- 172.21.15.83:0/2309084977 >> 172.21.15.83:0/2309084977 conn(0x7f738c09d510 msgr2=0x7f738c131fc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:39:05.063 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.051+0000 7f7393848640 1 -- 172.21.15.83:0/2309084977 shutdown_connections 2024-09-14T03:39:05.063 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.051+0000 7f7393848640 1 -- 172.21.15.83:0/2309084977 wait complete. 2024-09-14T03:39:05.064 INFO:teuthology.orchestra.run.smithi083.stdout:47244640260 2024-09-14T03:39:05.064 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.3 2024-09-14T03:39:05.065 INFO:teuthology.orchestra.run.smithi083.stdout:47244640260 2024-09-14T03:39:05.065 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.1 2024-09-14T03:39:05.066 INFO:teuthology.orchestra.run.smithi083.stdout:47244640260 2024-09-14T03:39:05.066 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.2 2024-09-14T03:39:05.067 INFO:teuthology.orchestra.run.smithi083.stdout:38654705668 2024-09-14T03:39:05.067 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.0 2024-09-14T03:39:05.211 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.209+0000 7fc115454640 1 -- 172.21.15.83:0/1172885250 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fc1101334f0 msgr2=0x7fc1101358e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:05.211 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.209+0000 7fc115454640 1 --2- 172.21.15.83:0/1172885250 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fc1101334f0 0x7fc1101358e0 secure :-1 s=READY pgs=66 cs=0 l=1 rev1=1 crypto rx=0x7fc108005fd0 tx=0x7fc10802a780 comp rx=0 tx=0).stop 2024-09-14T03:39:05.211 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.209+0000 7fc115454640 1 -- 172.21.15.83:0/1172885250 shutdown_connections 2024-09-14T03:39:05.211 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.209+0000 7fc115454640 1 --2- 172.21.15.83:0/1172885250 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fc1101334f0 0x7fc1101358e0 unknown :-1 s=CLOSED pgs=66 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:05.212 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.209+0000 7fc115454640 1 --2- 172.21.15.83:0/1172885250 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fc11008d270 0x7fc110132fb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:05.212 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.209+0000 7fc115454640 1 -- 172.21.15.83:0/1172885250 >> 172.21.15.83:0/1172885250 conn(0x7fc1100a0c80 msgr2=0x7fc11008b000 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:39:05.212 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.209+0000 7fc115454640 1 -- 172.21.15.83:0/1172885250 shutdown_connections 2024-09-14T03:39:05.212 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.209+0000 7fc115454640 1 -- 172.21.15.83:0/1172885250 wait complete. 2024-09-14T03:39:05.213 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.210+0000 7fc115454640 1 Processor -- start 2024-09-14T03:39:05.214 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.210+0000 7fc115454640 1 -- start start 2024-09-14T03:39:05.214 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.211+0000 7fc115454640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fc11008c780 0x7fc11008f240 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:39:05.214 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.211+0000 7fc115454640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fc11008d270 0x7fc11008f780 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:39:05.214 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.211+0000 7fc115454640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fc1101334f0 0x7fc11008fcc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:39:05.214 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.211+0000 7fc115454640 1 -- --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7fc1101368f0 con 0x7fc11008d270 2024-09-14T03:39:05.214 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.211+0000 7fc115454640 1 -- --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7fc110136770 con 0x7fc11008c780 2024-09-14T03:39:05.214 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.211+0000 7fc115454640 1 -- --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7fc110136a70 con 0x7fc1101334f0 2024-09-14T03:39:05.214 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.211+0000 7fc10f7fe640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fc1101334f0 0x7fc11008fcc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:39:05.214 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.211+0000 7fc10effd640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fc11008c780 0x7fc11008f240 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:39:05.214 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.211+0000 7fc10effd640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fc11008c780 0x7fc11008f240 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3301/0 says I am v2:172.21.15.83:60552/0 (socket says 172.21.15.83:60552) 2024-09-14T03:39:05.214 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.211+0000 7fc10effd640 1 -- 172.21.15.83:0/1434516659 learned_addr learned my addr 172.21.15.83:0/1434516659 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:39:05.215 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.211+0000 7fc10e7fc640 1 --2- 172.21.15.83:0/1434516659 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fc11008d270 0x7fc11008f780 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:39:05.215 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.212+0000 7fc10e7fc640 1 -- 172.21.15.83:0/1434516659 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fc1101334f0 msgr2=0x7fc11008fcc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:05.215 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.212+0000 7fc10e7fc640 1 --2- 172.21.15.83:0/1434516659 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fc1101334f0 0x7fc11008fcc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:05.215 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.212+0000 7fc10e7fc640 1 -- 172.21.15.83:0/1434516659 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fc11008c780 msgr2=0x7fc11008f240 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:05.215 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.212+0000 7fc10e7fc640 1 --2- 172.21.15.83:0/1434516659 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fc11008c780 0x7fc11008f240 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:05.215 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.212+0000 7fc10e7fc640 1 -- 172.21.15.83:0/1434516659 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fc110091a20 con 0x7fc11008d270 2024-09-14T03:39:05.215 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.212+0000 7fc10effd640 1 --2- 172.21.15.83:0/1434516659 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fc11008c780 0x7fc11008f240 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:39:05.215 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.212+0000 7fc10e7fc640 1 --2- 172.21.15.83:0/1434516659 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fc11008d270 0x7fc11008f780 secure :-1 s=READY pgs=69 cs=0 l=1 rev1=1 crypto rx=0x7fc104005fd0 tx=0x7fc10400ee50 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:39:05.215 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.212+0000 7fc10f7fe640 1 --2- 172.21.15.83:0/1434516659 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fc1101334f0 0x7fc11008fcc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-09-14T03:39:05.216 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.213+0000 7fc0f77fe640 1 -- 172.21.15.83:0/1434516659 <== mon.0 v2:172.21.15.83:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc10400cd00 con 0x7fc11008d270 2024-09-14T03:39:05.216 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.213+0000 7fc0f77fe640 1 -- 172.21.15.83:0/1434516659 <== mon.0 v2:172.21.15.83:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fc104004020 con 0x7fc11008d270 2024-09-14T03:39:05.216 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.213+0000 7fc0f77fe640 1 -- 172.21.15.83:0/1434516659 <== mon.0 v2:172.21.15.83:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc1040043d0 con 0x7fc11008d270 2024-09-14T03:39:05.216 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.213+0000 7fc115454640 1 -- 172.21.15.83:0/1434516659 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fc110090200 con 0x7fc11008d270 2024-09-14T03:39:05.217 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.213+0000 7fc115454640 1 -- 172.21.15.83:0/1434516659 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fc1101c9f30 con 0x7fc11008d270 2024-09-14T03:39:05.217 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.213+0000 7fc8cd40c640 1 -- 172.21.15.83:0/1745048765 <== mon.1 v2:172.21.15.83:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc8c0031b80 con 0x7fc8c806c6b0 2024-09-14T03:39:05.217 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.214+0000 7fc8d0672640 1 -- 172.21.15.83:0/1745048765 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fc8c806c6b0 msgr2=0x7fc8c81337d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:05.217 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.214+0000 7fc8d0672640 1 --2- 172.21.15.83:0/1745048765 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fc8c806c6b0 0x7fc8c81337d0 secure :-1 s=READY pgs=41 cs=0 l=1 rev1=1 crypto rx=0x7fc8c0009c10 tx=0x7fc8c002a680 comp rx=0 tx=0).stop 2024-09-14T03:39:05.217 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.214+0000 7fc8d0672640 1 -- 172.21.15.83:0/1745048765 shutdown_connections 2024-09-14T03:39:05.218 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.214+0000 7f649dffb640 1 -- 172.21.15.83:0/799547007 <== mon.2 v2:172.21.15.83:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f649802ad10 con 0x7f64a0094f10 2024-09-14T03:39:05.218 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.214+0000 7fc8d0672640 1 --2- 172.21.15.83:0/1745048765 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fc8c806c6b0 0x7fc8c81337d0 unknown :-1 s=CLOSED pgs=41 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:05.218 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.214+0000 7fc115454640 1 -- 172.21.15.83:0/1434516659 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fc110091d60 con 0x7fc11008d270 2024-09-14T03:39:05.219 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.214+0000 7fc8d0672640 1 --2- 172.21.15.83:0/1745048765 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fc8c806c2d0 0x7fc8c8131280 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:05.219 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.214+0000 7fc8d0672640 1 -- 172.21.15.83:0/1745048765 >> 172.21.15.83:0/1745048765 conn(0x7fc8c809d510 msgr2=0x7fc8c809d920 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:39:05.219 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.214+0000 7fc8d0672640 1 -- 172.21.15.83:0/1745048765 shutdown_connections 2024-09-14T03:39:05.219 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.214+0000 7fc8d0672640 1 -- 172.21.15.83:0/1745048765 wait complete. 2024-09-14T03:39:05.219 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.214+0000 7fc0f77fe640 1 -- 172.21.15.83:0/1434516659 <== mon.0 v2:172.21.15.83:3300/0 4 ==== mgrmap(e 5) ==== 55999+0+0 (secure 0 0 0) 0x7fc1040035f0 con 0x7fc11008d270 2024-09-14T03:39:05.220 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.215+0000 7f64a55a5640 1 -- 172.21.15.83:0/799547007 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f64a0094f10 msgr2=0x7f64a00952f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:05.220 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.215+0000 7f64a55a5640 1 --2- 172.21.15.83:0/799547007 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f64a0094f10 0x7f64a00952f0 secure :-1 s=READY pgs=43 cs=0 l=1 rev1=1 crypto rx=0x7f649800d910 tx=0x7f649802a1b0 comp rx=0 tx=0).stop 2024-09-14T03:39:05.220 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.215+0000 7f64a55a5640 1 -- 172.21.15.83:0/799547007 shutdown_connections 2024-09-14T03:39:05.220 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.215+0000 7f64a55a5640 1 --2- 172.21.15.83:0/799547007 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f64a0095830 0x7f64a012a9b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:05.221 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.215+0000 7f64a55a5640 1 --2- 172.21.15.83:0/799547007 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f64a0094f10 0x7f64a00952f0 unknown :-1 s=CLOSED pgs=43 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:05.221 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.215+0000 7f64a55a5640 1 -- 172.21.15.83:0/799547007 >> 172.21.15.83:0/799547007 conn(0x7f64a009b590 msgr2=0x7f64a009b9a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:39:05.221 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.215+0000 7f64a55a5640 1 -- 172.21.15.83:0/799547007 shutdown_connections 2024-09-14T03:39:05.221 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.215+0000 7fc0f77fe640 1 -- 172.21.15.83:0/1434516659 <== mon.0 v2:172.21.15.83:3300/0 5 ==== osd_map(18..18 src has 1..18) ==== 2868+0+0 (secure 0 0 0) 0x7fc10405e0b0 con 0x7fc11008d270 2024-09-14T03:39:05.222 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.215+0000 7f64a55a5640 1 -- 172.21.15.83:0/799547007 wait complete. 2024-09-14T03:39:05.223 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.215+0000 7f0ea9644640 1 -- 172.21.15.83:0/4186773741 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f0ea4070b10 msgr2=0x7f0ea412fe40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:05.223 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.215+0000 7f0ea9644640 1 --2- 172.21.15.83:0/4186773741 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f0ea4070b10 0x7f0ea412fe40 secure :-1 s=READY pgs=71 cs=0 l=1 rev1=1 crypto rx=0x7f0e98009830 tx=0x7f0e9802a3f0 comp rx=0 tx=0).stop 2024-09-14T03:39:05.224 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.215+0000 7f0ea9644640 1 -- 172.21.15.83:0/4186773741 shutdown_connections 2024-09-14T03:39:05.224 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.215+0000 7f0ea9644640 1 --2- 172.21.15.83:0/4186773741 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f0ea4130380 0x7f0ea4132770 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:05.224 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.215+0000 7f0ea9644640 1 --2- 172.21.15.83:0/4186773741 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f0ea4070b10 0x7f0ea412fe40 unknown :-1 s=CLOSED pgs=71 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:05.224 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.215+0000 7f0ea9644640 1 -- 172.21.15.83:0/4186773741 >> 172.21.15.83:0/4186773741 conn(0x7f0ea409cd80 msgr2=0x7f0ea409d190 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:39:05.224 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.215+0000 7f0ea9644640 1 -- 172.21.15.83:0/4186773741 shutdown_connections 2024-09-14T03:39:05.224 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.215+0000 7f0ea9644640 1 -- 172.21.15.83:0/4186773741 wait complete. 2024-09-14T03:39:05.225 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.216+0000 7fc8d0672640 1 Processor -- start 2024-09-14T03:39:05.225 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.216+0000 7fc8d0672640 1 -- start start 2024-09-14T03:39:05.226 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.216+0000 7f0ea9644640 1 Processor -- start 2024-09-14T03:39:05.226 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.216+0000 7f64a55a5640 1 Processor -- start 2024-09-14T03:39:05.227 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.216+0000 7f0ea9644640 1 -- start start 2024-09-14T03:39:05.227 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.217+0000 7f64a55a5640 1 -- start start 2024-09-14T03:39:05.228 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.217+0000 7fc8d0672640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fc8c806c2d0 0x7fc8c81221b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:39:05.228 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.217+0000 7fc8d0672640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fc8c806c6b0 0x7fc8c81226f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:39:05.228 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.217+0000 7fc8ce40e640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fc8c806c2d0 0x7fc8c81221b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:39:05.228 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.217+0000 7fc8ce40e640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fc8c806c2d0 0x7fc8c81221b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3300/0 says I am v2:172.21.15.83:35936/0 (socket says 172.21.15.83:35936) 2024-09-14T03:39:05.229 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.217+0000 7fc8d0672640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fc8c80937e0 0x7fc8c8122c30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:39:05.229 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.217+0000 7fc8cdc0d640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fc8c806c6b0 0x7fc8c81226f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:39:05.229 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.217+0000 7fc8cdc0d640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fc8c806c6b0 0x7fc8c81226f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3302/0 says I am v2:172.21.15.83:32912/0 (socket says 172.21.15.83:32912) 2024-09-14T03:39:05.229 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.217+0000 7fc8ce40e640 1 -- 172.21.15.83:0/1316478821 learned_addr learned my addr 172.21.15.83:0/1316478821 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:39:05.229 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.217+0000 7f0ea9644640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f0ea4070b10 0x7f0ea4126800 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:39:05.230 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.217+0000 7fc8d0672640 1 -- 172.21.15.83:0/1316478821 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7fc8c8134840 con 0x7fc8c806c2d0 2024-09-14T03:39:05.230 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.217+0000 7fc8d0672640 1 -- 172.21.15.83:0/1316478821 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7fc8c81346c0 con 0x7fc8c80937e0 2024-09-14T03:39:05.230 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.217+0000 7fc8cec0f640 1 --2- 172.21.15.83:0/1316478821 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fc8c80937e0 0x7fc8c8122c30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:39:05.231 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.217+0000 7fc8d0672640 1 -- 172.21.15.83:0/1316478821 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7fc8c81349c0 con 0x7fc8c806c6b0 2024-09-14T03:39:05.231 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.217+0000 7f64a55a5640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f64a0094f10 0x7f64a01bfa40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:39:05.231 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.217+0000 7f64a55a5640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f64a0095830 0x7f64a01bff80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:39:05.231 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.217+0000 7f64a55a5640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f64a012aef0 0x7f64a0071590 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:39:05.231 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.217+0000 7f64a55a5640 1 -- --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7f64a006bf90 con 0x7f64a0094f10 2024-09-14T03:39:05.231 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.217+0000 7f64a55a5640 1 -- --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7f64a006be10 con 0x7f64a0095830 2024-09-14T03:39:05.232 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.217+0000 7f64a55a5640 1 -- --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7f64a006c110 con 0x7f64a012aef0 2024-09-14T03:39:05.232 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.217+0000 7f0ea9644640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f0ea40963e0 0x7f0ea4126d40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:39:05.232 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.217+0000 7f0ea9644640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f0ea4130380 0x7f0ea4127280 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:39:05.232 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.217+0000 7f0ea9644640 1 -- --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7f0ea4133800 con 0x7f0ea40963e0 2024-09-14T03:39:05.232 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.217+0000 7f0ea9644640 1 -- --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7f0ea4133680 con 0x7f0ea4130380 2024-09-14T03:39:05.233 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.217+0000 7f0ea9644640 1 -- --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7f0ea4133980 con 0x7f0ea4070b10 2024-09-14T03:39:05.233 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.217+0000 7f0ea2ffd640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f0ea4070b10 0x7f0ea4126800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:39:05.233 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.217+0000 7f0ea2ffd640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f0ea4070b10 0x7f0ea4126800 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3302/0 says I am v2:172.21.15.83:32920/0 (socket says 172.21.15.83:32920) 2024-09-14T03:39:05.233 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.217+0000 7f0ea37fe640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f0ea4130380 0x7f0ea4127280 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:39:05.233 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.217+0000 7f0ea27fc640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f0ea40963e0 0x7f0ea4126d40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:39:05.233 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.217+0000 7f0ea2ffd640 1 -- 172.21.15.83:0/4182299752 learned_addr learned my addr 172.21.15.83:0/4182299752 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:39:05.233 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.217+0000 7f649f7fe640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f64a012aef0 0x7f64a0071590 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:39:05.234 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.217+0000 7f649e7fc640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f64a0095830 0x7f64a01bff80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:39:05.234 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.217+0000 7f649f7fe640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f64a012aef0 0x7f64a0071590 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3302/0 says I am v2:172.21.15.83:32932/0 (socket says 172.21.15.83:32932) 2024-09-14T03:39:05.234 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.217+0000 7f649f7fe640 1 -- 172.21.15.83:0/2696885077 learned_addr learned my addr 172.21.15.83:0/2696885077 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:39:05.234 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.218+0000 7fc8ce40e640 1 -- 172.21.15.83:0/1316478821 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fc8c806c6b0 msgr2=0x7fc8c81226f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:05.234 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.218+0000 7f649f7fe640 1 -- 172.21.15.83:0/2696885077 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f64a0095830 msgr2=0x7f64a01bff80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:05.234 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.218+0000 7f649effd640 1 --2- 172.21.15.83:0/2696885077 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f64a0094f10 0x7f64a01bfa40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:39:05.235 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.218+0000 7f0ea37fe640 1 -- 172.21.15.83:0/4182299752 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f0ea4070b10 msgr2=0x7f0ea4126800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:05.235 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.218+0000 7f0ea37fe640 1 --2- 172.21.15.83:0/4182299752 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f0ea4070b10 0x7f0ea4126800 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:05.235 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.218+0000 7f0ea37fe640 1 -- 172.21.15.83:0/4182299752 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f0ea40963e0 msgr2=0x7f0ea4126d40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:05.235 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.218+0000 7f0ea37fe640 1 --2- 172.21.15.83:0/4182299752 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f0ea40963e0 0x7f0ea4126d40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:05.235 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.218+0000 7f0ea37fe640 1 -- 172.21.15.83:0/4182299752 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f0ea406e6d0 con 0x7f0ea4130380 2024-09-14T03:39:05.235 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.218+0000 7f0ea37fe640 1 --2- 172.21.15.83:0/4182299752 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f0ea4130380 0x7f0ea4127280 secure :-1 s=READY pgs=42 cs=0 l=1 rev1=1 crypto rx=0x7f0e9c005fd0 tx=0x7f0e9c00ee60 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:39:05.236 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.218+0000 7f649f7fe640 1 --2- 172.21.15.83:0/2696885077 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f64a0095830 0x7f64a01bff80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:05.236 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.218+0000 7f649f7fe640 1 -- 172.21.15.83:0/2696885077 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f64a0094f10 msgr2=0x7f64a01bfa40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:05.236 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.218+0000 7f649f7fe640 1 --2- 172.21.15.83:0/2696885077 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f64a0094f10 0x7f64a01bfa40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:05.236 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.218+0000 7f649f7fe640 1 -- 172.21.15.83:0/2696885077 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f64a008c450 con 0x7f64a012aef0 2024-09-14T03:39:05.236 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.218+0000 7f0e8b7fe640 1 -- 172.21.15.83:0/4182299752 <== mon.1 v2:172.21.15.83:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0e9c00cd80 con 0x7f0ea4130380 2024-09-14T03:39:05.236 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.218+0000 7f0e8b7fe640 1 -- 172.21.15.83:0/4182299752 <== mon.1 v2:172.21.15.83:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f0e9c004c10 con 0x7f0ea4130380 2024-09-14T03:39:05.237 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.218+0000 7f649e7fc640 1 --2- 172.21.15.83:0/2696885077 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f64a0095830 0x7f64a01bff80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:39:05.237 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.218+0000 7f0ea9644640 1 -- 172.21.15.83:0/4182299752 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f0ea4128e00 con 0x7f0ea4130380 2024-09-14T03:39:05.237 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.218+0000 7f649effd640 1 --2- 172.21.15.83:0/2696885077 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f64a0094f10 0x7f64a01bfa40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-09-14T03:39:05.237 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.218+0000 7fc8ce40e640 1 --2- 172.21.15.83:0/1316478821 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fc8c806c6b0 0x7fc8c81226f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:05.237 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.218+0000 7fc8ce40e640 1 -- 172.21.15.83:0/1316478821 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fc8c80937e0 msgr2=0x7fc8c8122c30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:05.238 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.218+0000 7fc8ce40e640 1 --2- 172.21.15.83:0/1316478821 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fc8c80937e0 0x7fc8c8122c30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:05.238 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.218+0000 7f0ea9644640 1 -- 172.21.15.83:0/4182299752 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7f0ea41c7c90 con 0x7f0ea4130380 2024-09-14T03:39:05.238 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.218+0000 7f0e8b7fe640 1 -- 172.21.15.83:0/4182299752 <== mon.1 v2:172.21.15.83:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0e9c005020 con 0x7f0ea4130380 2024-09-14T03:39:05.238 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.218+0000 7fc8ce40e640 1 -- 172.21.15.83:0/1316478821 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fc8c8090040 con 0x7fc8c806c2d0 2024-09-14T03:39:05.239 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.218+0000 7fc8cec0f640 1 --2- 172.21.15.83:0/1316478821 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fc8c80937e0 0x7fc8c8122c30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:39:05.239 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.219+0000 7fc8cdc0d640 1 --2- 172.21.15.83:0/1316478821 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fc8c806c6b0 0x7fc8c81226f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:39:05.239 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.219+0000 7f649f7fe640 1 --2- 172.21.15.83:0/2696885077 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f64a012aef0 0x7f64a0071590 secure :-1 s=READY pgs=44 cs=0 l=1 rev1=1 crypto rx=0x7f648800c780 tx=0x7f648800cc40 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:39:05.239 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.219+0000 7f64977fe640 1 -- 172.21.15.83:0/2696885077 <== mon.2 v2:172.21.15.83:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6488004b10 con 0x7f64a012aef0 2024-09-14T03:39:05.239 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.219+0000 7f64a55a5640 1 -- 172.21.15.83:0/2696885077 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f64a008bae0 con 0x7f64a012aef0 2024-09-14T03:39:05.239 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.219+0000 7f64977fe640 1 -- 172.21.15.83:0/2696885077 <== mon.2 v2:172.21.15.83:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f6488004da0 con 0x7f64a012aef0 2024-09-14T03:39:05.240 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.219+0000 7fc8ce40e640 1 --2- 172.21.15.83:0/1316478821 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fc8c806c2d0 0x7fc8c81221b0 secure :-1 s=READY pgs=72 cs=0 l=1 rev1=1 crypto rx=0x7fc8c400c780 tx=0x7fc8c400cb80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:39:05.240 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.219+0000 7f64977fe640 1 -- 172.21.15.83:0/2696885077 <== mon.2 v2:172.21.15.83:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6488004b10 con 0x7f64a012aef0 2024-09-14T03:39:05.240 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.219+0000 7f64a55a5640 1 -- 172.21.15.83:0/2696885077 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7f64a0071ad0 con 0x7f64a012aef0 2024-09-14T03:39:05.241 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.219+0000 7f0ea9644640 1 -- 172.21.15.83:0/4182299752 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f0ea40967c0 con 0x7f0ea4130380 2024-09-14T03:39:05.241 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.220+0000 7f0e8b7fe640 1 -- 172.21.15.83:0/4182299752 <== mon.1 v2:172.21.15.83:3301/0 4 ==== mgrmap(e 5) ==== 55999+0+0 (secure 0 0 0) 0x7f0e9c0051c0 con 0x7f0ea4130380 2024-09-14T03:39:05.241 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.220+0000 7f0e8b7fe640 1 -- 172.21.15.83:0/4182299752 <== mon.1 v2:172.21.15.83:3301/0 5 ==== osd_map(18..18 src has 1..18) ==== 2868+0+0 (secure 0 0 0) 0x7f0e9c05e690 con 0x7f0ea4130380 2024-09-14T03:39:05.241 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.220+0000 7fc0f77fe640 1 -- 172.21.15.83:0/1434516659 <== mon.0 v2:172.21.15.83:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+151260 (secure 0 0 0) 0x7fc110091d60 con 0x7fc11008d270 2024-09-14T03:39:05.242 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.220+0000 7fc8beffd640 1 -- 172.21.15.83:0/1316478821 <== mon.0 v2:172.21.15.83:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc8c4017070 con 0x7fc8c806c2d0 2024-09-14T03:39:05.242 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.220+0000 7fc8beffd640 1 -- 172.21.15.83:0/1316478821 <== mon.0 v2:172.21.15.83:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fc8c4005360 con 0x7fc8c806c2d0 2024-09-14T03:39:05.242 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.221+0000 7fc8beffd640 1 -- 172.21.15.83:0/1316478821 <== mon.0 v2:172.21.15.83:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc8c400a8a0 con 0x7fc8c806c2d0 2024-09-14T03:39:05.242 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.221+0000 7fc8d0672640 1 -- 172.21.15.83:0/1316478821 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fc8c8090930 con 0x7fc8c806c2d0 2024-09-14T03:39:05.242 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.221+0000 7f64977fe640 1 -- 172.21.15.83:0/2696885077 <== mon.2 v2:172.21.15.83:3302/0 4 ==== mgrmap(e 5) ==== 55999+0+0 (secure 0 0 0) 0x7f648800a850 con 0x7f64a012aef0 2024-09-14T03:39:05.243 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.221+0000 7fc8d0672640 1 -- 172.21.15.83:0/1316478821 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fc8c8123350 con 0x7fc8c806c2d0 2024-09-14T03:39:05.243 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.221+0000 7f64977fe640 1 -- 172.21.15.83:0/2696885077 <== mon.2 v2:172.21.15.83:3302/0 5 ==== osd_map(18..18 src has 1..18) ==== 2868+0+0 (secure 0 0 0) 0x7f6488004910 con 0x7f64a012aef0 2024-09-14T03:39:05.243 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.222+0000 7f64a55a5640 1 -- 172.21.15.83:0/2696885077 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f64a00952f0 con 0x7f64a012aef0 2024-09-14T03:39:05.244 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.222+0000 7fc8beffd640 1 -- 172.21.15.83:0/1316478821 <== mon.0 v2:172.21.15.83:3300/0 4 ==== mgrmap(e 5) ==== 55999+0+0 (secure 0 0 0) 0x7fc8c401a070 con 0x7fc8c806c2d0 2024-09-14T03:39:05.244 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.222+0000 7fc8beffd640 1 -- 172.21.15.83:0/1316478821 <== mon.0 v2:172.21.15.83:3300/0 5 ==== osd_map(18..18 src has 1..18) ==== 2868+0+0 (secure 0 0 0) 0x7fc8c4063060 con 0x7fc8c806c2d0 2024-09-14T03:39:05.244 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.224+0000 7fc8d0672640 1 -- 172.21.15.83:0/1316478821 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fc8c806bbc0 con 0x7fc8c806c2d0 2024-09-14T03:39:05.244 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.225+0000 7f0e8b7fe640 1 -- 172.21.15.83:0/4182299752 <== mon.1 v2:172.21.15.83:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+151260 (secure 0 0 0) 0x7f0e9c05d7e0 con 0x7f0ea4130380 2024-09-14T03:39:05.245 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.228+0000 7fc8beffd640 1 -- 172.21.15.83:0/1316478821 <== mon.0 v2:172.21.15.83:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+151260 (secure 0 0 0) 0x7fc8c403be40 con 0x7fc8c806c2d0 2024-09-14T03:39:05.245 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.229+0000 7f64977fe640 1 -- 172.21.15.83:0/2696885077 <== mon.2 v2:172.21.15.83:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+151260 (secure 0 0 0) 0x7f648805ca30 con 0x7f64a012aef0 2024-09-14T03:39:05.373 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.371+0000 7fc115454640 1 -- 172.21.15.83:0/1434516659 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 0} v 0) -- 0x7fc11008cb60 con 0x7fc11008d270 2024-09-14T03:39:05.373 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.371+0000 7fc0f77fe640 1 -- 172.21.15.83:0/1434516659 <== mon.0 v2:172.21.15.83:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 0}]=0 v0) ==== 74+0+12 (secure 0 0 0) 0x7fc110091d60 con 0x7fc11008d270 2024-09-14T03:39:05.374 INFO:teuthology.orchestra.run.smithi083.stdout:38654705666 2024-09-14T03:39:05.376 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.373+0000 7fc115454640 1 -- 172.21.15.83:0/1434516659 >> v1:172.21.15.83:6800/1934165865 conn(0x7fc0fc048020 legacy=0x7fc0fc04a4e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:05.377 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.373+0000 7fc115454640 1 -- 172.21.15.83:0/1434516659 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fc11008d270 msgr2=0x7fc11008f780 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:05.377 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.373+0000 7fc115454640 1 --2- 172.21.15.83:0/1434516659 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fc11008d270 0x7fc11008f780 secure :-1 s=READY pgs=69 cs=0 l=1 rev1=1 crypto rx=0x7fc104005fd0 tx=0x7fc10400ee50 comp rx=0 tx=0).stop 2024-09-14T03:39:05.377 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.373+0000 7fc115454640 1 -- 172.21.15.83:0/1434516659 shutdown_connections 2024-09-14T03:39:05.377 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.373+0000 7fc115454640 1 --2- 172.21.15.83:0/1434516659 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fc1101334f0 0x7fc11008fcc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:05.377 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.373+0000 7fc115454640 1 --2- 172.21.15.83:0/1434516659 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fc11008d270 0x7fc11008f780 unknown :-1 s=CLOSED pgs=69 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:05.377 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.373+0000 7fc115454640 1 --2- 172.21.15.83:0/1434516659 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fc11008c780 0x7fc11008f240 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:05.377 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.373+0000 7fc115454640 1 -- 172.21.15.83:0/1434516659 >> 172.21.15.83:0/1434516659 conn(0x7fc1100a0c80 msgr2=0x7fc11006e040 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:39:05.377 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.373+0000 7fc115454640 1 -- 172.21.15.83:0/1434516659 shutdown_connections 2024-09-14T03:39:05.377 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.373+0000 7fc115454640 1 -- 172.21.15.83:0/1434516659 wait complete. 2024-09-14T03:39:05.378 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.376+0000 7f0ea9644640 1 -- 172.21.15.83:0/4182299752 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 2} v 0) -- 0x7f0ea4072e70 con 0x7f0ea4130380 2024-09-14T03:39:05.378 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.376+0000 7f0e8b7fe640 1 -- 172.21.15.83:0/4182299752 <== mon.1 v2:172.21.15.83:3301/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 2}]=0 v0) ==== 74+0+12 (secure 0 0 0) 0x7f0ea4072e70 con 0x7f0ea4130380 2024-09-14T03:39:05.378 INFO:teuthology.orchestra.run.smithi083.stdout:47244640259 2024-09-14T03:39:05.379 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.378+0000 7f64a55a5640 1 -- 172.21.15.83:0/2696885077 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 3} v 0) -- 0x7f64a0068c60 con 0x7f64a012aef0 2024-09-14T03:39:05.379 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.378+0000 7f64977fe640 1 -- 172.21.15.83:0/2696885077 <== mon.2 v2:172.21.15.83:3302/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 3}]=0 v0) ==== 74+0+12 (secure 0 0 0) 0x7f648803cb70 con 0x7f64a012aef0 2024-09-14T03:39:05.379 INFO:teuthology.orchestra.run.smithi083.stdout:47244640259 2024-09-14T03:39:05.380 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.378+0000 7fc8d0672640 1 -- 172.21.15.83:0/1316478821 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 1} v 0) -- 0x7fc8c8068c60 con 0x7fc8c806c2d0 2024-09-14T03:39:05.380 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.379+0000 7fc8beffd640 1 -- 172.21.15.83:0/1316478821 <== mon.0 v2:172.21.15.83:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 1}]=0 v0) ==== 74+0+12 (secure 0 0 0) 0x7fc8c403c0a0 con 0x7fc8c806c2d0 2024-09-14T03:39:05.380 INFO:teuthology.orchestra.run.smithi083.stdout:47244640259 2024-09-14T03:39:05.381 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.379+0000 7f0ea9644640 1 -- 172.21.15.83:0/4182299752 >> v1:172.21.15.83:6800/1934165865 conn(0x7f0e90047fd0 legacy=0x7f0e9004a490 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:05.381 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.379+0000 7f0ea9644640 1 -- 172.21.15.83:0/4182299752 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f0ea4130380 msgr2=0x7f0ea4127280 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:05.381 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.379+0000 7f0ea9644640 1 --2- 172.21.15.83:0/4182299752 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f0ea4130380 0x7f0ea4127280 secure :-1 s=READY pgs=42 cs=0 l=1 rev1=1 crypto rx=0x7f0e9c005fd0 tx=0x7f0e9c00ee60 comp rx=0 tx=0).stop 2024-09-14T03:39:05.381 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.379+0000 7f0ea9644640 1 -- 172.21.15.83:0/4182299752 shutdown_connections 2024-09-14T03:39:05.382 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.379+0000 7f0ea9644640 1 --2- 172.21.15.83:0/4182299752 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f0ea4130380 0x7f0ea4127280 unknown :-1 s=CLOSED pgs=42 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:05.382 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.379+0000 7f0ea9644640 1 --2- 172.21.15.83:0/4182299752 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f0ea40963e0 0x7f0ea4126d40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:05.382 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.379+0000 7f0ea9644640 1 --2- 172.21.15.83:0/4182299752 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f0ea4070b10 0x7f0ea4126800 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:05.382 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.379+0000 7f0ea9644640 1 -- 172.21.15.83:0/4182299752 >> 172.21.15.83:0/4182299752 conn(0x7f0ea409cd80 msgr2=0x7f0ea412bc10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:39:05.382 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.379+0000 7f0ea9644640 1 -- 172.21.15.83:0/4182299752 shutdown_connections 2024-09-14T03:39:05.382 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.379+0000 7f0ea9644640 1 -- 172.21.15.83:0/4182299752 wait complete. 2024-09-14T03:39:05.383 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.380+0000 7f64a55a5640 1 -- 172.21.15.83:0/2696885077 >> v1:172.21.15.83:6800/1934165865 conn(0x7f64840483d0 legacy=0x7f648404a890 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:05.383 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.380+0000 7f64a55a5640 1 -- 172.21.15.83:0/2696885077 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f64a012aef0 msgr2=0x7f64a0071590 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:05.383 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.380+0000 7f64a55a5640 1 --2- 172.21.15.83:0/2696885077 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f64a012aef0 0x7f64a0071590 secure :-1 s=READY pgs=44 cs=0 l=1 rev1=1 crypto rx=0x7f648800c780 tx=0x7f648800cc40 comp rx=0 tx=0).stop 2024-09-14T03:39:05.383 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.381+0000 7f64a55a5640 1 -- 172.21.15.83:0/2696885077 shutdown_connections 2024-09-14T03:39:05.383 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.381+0000 7f64a55a5640 1 --2- 172.21.15.83:0/2696885077 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f64a012aef0 0x7f64a0071590 unknown :-1 s=CLOSED pgs=44 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:05.383 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.381+0000 7f64a55a5640 1 --2- 172.21.15.83:0/2696885077 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f64a0095830 0x7f64a01bff80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:05.383 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.381+0000 7f64a55a5640 1 --2- 172.21.15.83:0/2696885077 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f64a0094f10 0x7f64a01bfa40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:05.383 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.381+0000 7f64a55a5640 1 -- 172.21.15.83:0/2696885077 >> 172.21.15.83:0/2696885077 conn(0x7f64a009b590 msgr2=0x7f64a0124340 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:39:05.383 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.381+0000 7fc8d0672640 1 -- 172.21.15.83:0/1316478821 >> v1:172.21.15.83:6800/1934165865 conn(0x7fc8ac047fd0 legacy=0x7fc8ac04a490 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:05.383 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.381+0000 7fc8d0672640 1 -- 172.21.15.83:0/1316478821 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fc8c806c2d0 msgr2=0x7fc8c81221b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:05.383 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.381+0000 7fc8d0672640 1 --2- 172.21.15.83:0/1316478821 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fc8c806c2d0 0x7fc8c81221b0 secure :-1 s=READY pgs=72 cs=0 l=1 rev1=1 crypto rx=0x7fc8c400c780 tx=0x7fc8c400cb80 comp rx=0 tx=0).stop 2024-09-14T03:39:05.384 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.381+0000 7f64a55a5640 1 -- 172.21.15.83:0/2696885077 shutdown_connections 2024-09-14T03:39:05.384 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.381+0000 7fc8d0672640 1 -- 172.21.15.83:0/1316478821 shutdown_connections 2024-09-14T03:39:05.384 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.381+0000 7fc8d0672640 1 --2- 172.21.15.83:0/1316478821 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fc8c80937e0 0x7fc8c8122c30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:05.384 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.381+0000 7fc8d0672640 1 --2- 172.21.15.83:0/1316478821 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fc8c806c6b0 0x7fc8c81226f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:05.384 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.381+0000 7fc8d0672640 1 --2- 172.21.15.83:0/1316478821 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fc8c806c2d0 0x7fc8c81221b0 unknown :-1 s=CLOSED pgs=72 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:05.384 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.381+0000 7fc8d0672640 1 -- 172.21.15.83:0/1316478821 >> 172.21.15.83:0/1316478821 conn(0x7fc8c809d510 msgr2=0x7fc8c80711a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:39:05.385 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.381+0000 7f64a55a5640 1 -- 172.21.15.83:0/2696885077 wait complete. 2024-09-14T03:39:05.385 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.381+0000 7fc8d0672640 1 -- 172.21.15.83:0/1316478821 shutdown_connections 2024-09-14T03:39:05.385 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:05.381+0000 7fc8d0672640 1 -- 172.21.15.83:0/1316478821 wait complete. 2024-09-14T03:39:05.389 INFO:tasks.ceph.ceph_manager.ceph:need seq 38654705668 got 38654705666 for osd.0 2024-09-14T03:39:05.395 INFO:tasks.ceph.ceph_manager.ceph:need seq 47244640260 got 47244640259 for osd.3 2024-09-14T03:39:05.399 INFO:tasks.ceph.ceph_manager.ceph:need seq 47244640260 got 47244640259 for osd.2 2024-09-14T03:39:05.399 INFO:tasks.ceph.ceph_manager.ceph:need seq 47244640260 got 47244640259 for osd.1 2024-09-14T03:39:06.391 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.0 2024-09-14T03:39:06.396 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.3 2024-09-14T03:39:06.400 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.2 2024-09-14T03:39:06.400 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.1 2024-09-14T03:39:06.506 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.501+0000 7f4c2e383640 1 -- 172.21.15.83:0/2928425242 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f4c2806e4a0 msgr2=0x7f4c2806e8a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:06.507 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.501+0000 7f4c2e383640 1 --2- 172.21.15.83:0/2928425242 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f4c2806e4a0 0x7f4c2806e8a0 secure :-1 s=READY pgs=45 cs=0 l=1 rev1=1 crypto rx=0x7f4c1c009860 tx=0x7f4c1c02a390 comp rx=0 tx=0).stop 2024-09-14T03:39:06.507 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.502+0000 7f4c2e383640 1 -- 172.21.15.83:0/2928425242 shutdown_connections 2024-09-14T03:39:06.507 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.502+0000 7f4c2e383640 1 --2- 172.21.15.83:0/2928425242 >> v2:172.21.15.83:3300/0 conn(0x7f4c2806ede0 0x7f4c28132e60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:06.507 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.502+0000 7f4c2e383640 1 --2- 172.21.15.83:0/2928425242 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f4c2806e4a0 0x7f4c2806e8a0 unknown :-1 s=CLOSED pgs=45 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:06.507 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.502+0000 7f4c2e383640 1 --2- 172.21.15.83:0/2928425242 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f4c28096400 0x7f4c280967e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:06.507 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.502+0000 7f4c2e383640 1 -- 172.21.15.83:0/2928425242 >> 172.21.15.83:0/2928425242 conn(0x7f4c2809cdf0 msgr2=0x7f4c2809d200 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:39:06.507 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.502+0000 7f4c2e383640 1 -- 172.21.15.83:0/2928425242 shutdown_connections 2024-09-14T03:39:06.508 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.502+0000 7f4c2e383640 1 -- 172.21.15.83:0/2928425242 wait complete. 2024-09-14T03:39:06.508 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.502+0000 7f4c2e383640 1 Processor -- start 2024-09-14T03:39:06.508 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.503+0000 7f4c2e383640 1 -- start start 2024-09-14T03:39:06.508 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.503+0000 7f4c2e383640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f4c2806e4a0 0x7f4c281c3e80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:39:06.508 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.503+0000 7f4c2e383640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f4c2806ede0 0x7f4c281c43c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:39:06.508 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.503+0000 7f4c2e383640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f4c28096400 0x7f4c2812ab70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:39:06.508 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.503+0000 7f4c2e383640 1 -- --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7f4c28133870 con 0x7f4c2806ede0 2024-09-14T03:39:06.508 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.503+0000 7f4c2e383640 1 -- --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7f4c281336f0 con 0x7f4c2806e4a0 2024-09-14T03:39:06.509 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.503+0000 7f4c2e383640 1 -- --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7f4c281339f0 con 0x7f4c28096400 2024-09-14T03:39:06.509 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.503+0000 7f4c2c920640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f4c28096400 0x7f4c2812ab70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:39:06.509 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.503+0000 7f4c2c920640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f4c28096400 0x7f4c2812ab70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3302/0 says I am v2:172.21.15.83:32948/0 (socket says 172.21.15.83:32948) 2024-09-14T03:39:06.509 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.503+0000 7f4c2c920640 1 -- 172.21.15.83:0/1851606781 learned_addr learned my addr 172.21.15.83:0/1851606781 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:39:06.509 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.503+0000 7f4c277fe640 1 --2- 172.21.15.83:0/1851606781 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f4c2806ede0 0x7f4c281c43c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:39:06.510 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.503+0000 7f4c27fff640 1 --2- 172.21.15.83:0/1851606781 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f4c2806e4a0 0x7f4c281c3e80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:39:06.510 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.504+0000 7f4c2c920640 1 -- 172.21.15.83:0/1851606781 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f4c2806e4a0 msgr2=0x7f4c281c3e80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:06.510 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.504+0000 7f4c2c920640 1 --2- 172.21.15.83:0/1851606781 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f4c2806e4a0 0x7f4c281c3e80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:06.510 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.504+0000 7f4c2c920640 1 -- 172.21.15.83:0/1851606781 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f4c2806ede0 msgr2=0x7f4c281c43c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:06.510 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.504+0000 7f4c2c920640 1 --2- 172.21.15.83:0/1851606781 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f4c2806ede0 0x7f4c281c43c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:06.511 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.504+0000 7f4c2c920640 1 -- 172.21.15.83:0/1851606781 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f4c28071030 con 0x7f4c28096400 2024-09-14T03:39:06.511 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.504+0000 7f4c27fff640 1 --2- 172.21.15.83:0/1851606781 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f4c2806e4a0 0x7f4c281c3e80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-09-14T03:39:06.511 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.504+0000 7f4c2c920640 1 --2- 172.21.15.83:0/1851606781 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f4c28096400 0x7f4c2812ab70 secure :-1 s=READY pgs=46 cs=0 l=1 rev1=1 crypto rx=0x7f4c20009fd0 tx=0x7f4c2000c800 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:39:06.511 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.504+0000 7f4c24ff9640 1 -- 172.21.15.83:0/1851606781 <== mon.2 v2:172.21.15.83:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4c20004b20 con 0x7f4c28096400 2024-09-14T03:39:06.511 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.504+0000 7f4c24ff9640 1 -- 172.21.15.83:0/1851606781 <== mon.2 v2:172.21.15.83:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f4c20004dd0 con 0x7f4c28096400 2024-09-14T03:39:06.511 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.504+0000 7f4c2e383640 1 -- 172.21.15.83:0/1851606781 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f4c2806f7c0 con 0x7f4c28096400 2024-09-14T03:39:06.511 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.504+0000 7f4c24ff9640 1 -- 172.21.15.83:0/1851606781 <== mon.2 v2:172.21.15.83:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4c20003bd0 con 0x7f4c28096400 2024-09-14T03:39:06.511 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.504+0000 7f4c2e383640 1 -- 172.21.15.83:0/1851606781 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7f4c280701c0 con 0x7f4c28096400 2024-09-14T03:39:06.513 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.505+0000 7f4c24ff9640 1 -- 172.21.15.83:0/1851606781 <== mon.2 v2:172.21.15.83:3302/0 4 ==== mgrmap(e 5) ==== 55999+0+0 (secure 0 0 0) 0x7f4c2001d020 con 0x7f4c28096400 2024-09-14T03:39:06.513 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.506+0000 7f4c2e383640 1 -- 172.21.15.83:0/1851606781 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f4c2806e8a0 con 0x7f4c28096400 2024-09-14T03:39:06.513 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.506+0000 7f4c24ff9640 1 -- 172.21.15.83:0/1851606781 <== mon.2 v2:172.21.15.83:3302/0 5 ==== osd_map(18..18 src has 1..18) ==== 2868+0+0 (secure 0 0 0) 0x7f4c2005e000 con 0x7f4c28096400 2024-09-14T03:39:06.514 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.510+0000 7f4c24ff9640 1 -- 172.21.15.83:0/1851606781 <== mon.2 v2:172.21.15.83:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+151260 (secure 0 0 0) 0x7f4c2005d150 con 0x7f4c28096400 2024-09-14T03:39:06.545 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.543+0000 7f8a0d0e0640 1 -- 172.21.15.83:0/897395792 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f8a0812e3a0 msgr2=0x7f8a0812e7d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:06.545 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.543+0000 7f8a0d0e0640 1 --2- 172.21.15.83:0/897395792 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f8a0812e3a0 0x7f8a0812e7d0 secure :-1 s=READY pgs=75 cs=0 l=1 rev1=1 crypto rx=0x7f8a00005fd0 tx=0x7f8a0002a930 comp rx=0 tx=0).stop 2024-09-14T03:39:06.545 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.543+0000 7f8a0d0e0640 1 -- 172.21.15.83:0/897395792 shutdown_connections 2024-09-14T03:39:06.545 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.543+0000 7f8a0d0e0640 1 --2- 172.21.15.83:0/897395792 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f8a0812e3a0 0x7f8a0812e7d0 unknown :-1 s=CLOSED pgs=75 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:06.545 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.543+0000 7f8a0d0e0640 1 --2- 172.21.15.83:0/897395792 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f8a0808c780 0x7f8a0808cb60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:06.545 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.543+0000 7f8a0d0e0640 1 -- 172.21.15.83:0/897395792 >> 172.21.15.83:0/897395792 conn(0x7f8a080a0c80 msgr2=0x7f8a0808b000 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:39:06.545 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.543+0000 7f8a0d0e0640 1 -- 172.21.15.83:0/897395792 shutdown_connections 2024-09-14T03:39:06.545 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.543+0000 7f8a0d0e0640 1 -- 172.21.15.83:0/897395792 wait complete. 2024-09-14T03:39:06.546 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.543+0000 7f90945d4640 1 -- 172.21.15.83:0/1577073825 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f908c09ba20 msgr2=0x7f908c12a9e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:06.546 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.543+0000 7f90945d4640 1 --2- 172.21.15.83:0/1577073825 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f908c09ba20 0x7f908c12a9e0 secure :-1 s=READY pgs=76 cs=0 l=1 rev1=1 crypto rx=0x7f907c009830 tx=0x7f907c02a370 comp rx=0 tx=0).stop 2024-09-14T03:39:06.546 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.543+0000 7f90945d4640 1 -- 172.21.15.83:0/1577073825 shutdown_connections 2024-09-14T03:39:06.546 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.543+0000 7f90945d4640 1 --2- 172.21.15.83:0/1577073825 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f908c12f6f0 0x7f908c131ae0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:06.546 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.543+0000 7f90945d4640 1 --2- 172.21.15.83:0/1577073825 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f908c09ba20 0x7f908c12a9e0 unknown :-1 s=CLOSED pgs=76 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:06.546 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.543+0000 7f90945d4640 1 --2- 172.21.15.83:0/1577073825 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f908c09b640 0x7f908c12a4a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:06.547 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.543+0000 7fe97dfbc640 1 -- 172.21.15.83:0/2523082346 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fe960004800 msgr2=0x7fe960004c80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:06.547 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.543+0000 7fe97dfbc640 1 --2- 172.21.15.83:0/2523082346 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fe960004800 0x7fe960004c80 secure :-1 s=READY pgs=77 cs=0 l=1 rev1=1 crypto rx=0x7fe970005fd0 tx=0x7fe960005830 comp rx=0 tx=0).stop 2024-09-14T03:39:06.547 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.543+0000 7f90945d4640 1 -- 172.21.15.83:0/1577073825 >> 172.21.15.83:0/1577073825 conn(0x7f908c09a610 msgr2=0x7f908c09aa20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:39:06.548 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.543+0000 7fe97dfbc640 1 -- 172.21.15.83:0/2523082346 shutdown_connections 2024-09-14T03:39:06.548 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.543+0000 7fe97dfbc640 1 --2- 172.21.15.83:0/2523082346 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fe960004800 0x7fe960004c80 unknown :-1 s=CLOSED pgs=77 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:06.548 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.543+0000 7fe97dfbc640 1 --2- 172.21.15.83:0/2523082346 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fe97812c1d0 0x7fe960003f90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:06.548 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.543+0000 7fe97dfbc640 1 --2- 172.21.15.83:0/2523082346 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fe978092880 0x7fe9600039b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:06.548 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.543+0000 7fe97dfbc640 1 -- 172.21.15.83:0/2523082346 >> 172.21.15.83:0/2523082346 conn(0x7fe97809c630 msgr2=0x7fe97809ca40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:39:06.548 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.544+0000 7fe97dfbc640 1 -- 172.21.15.83:0/2523082346 shutdown_connections 2024-09-14T03:39:06.548 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.544+0000 7fe97dfbc640 1 -- 172.21.15.83:0/2523082346 wait complete. 2024-09-14T03:39:06.549 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.543+0000 7f90945d4640 1 -- 172.21.15.83:0/1577073825 shutdown_connections 2024-09-14T03:39:06.549 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.543+0000 7f90945d4640 1 -- 172.21.15.83:0/1577073825 wait complete. 2024-09-14T03:39:06.549 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.544+0000 7f90945d4640 1 Processor -- start 2024-09-14T03:39:06.549 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.544+0000 7fe97dfbc640 1 Processor -- start 2024-09-14T03:39:06.549 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.544+0000 7f90945d4640 1 -- start start 2024-09-14T03:39:06.550 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.544+0000 7f8a0d0e0640 1 Processor -- start 2024-09-14T03:39:06.550 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.544+0000 7f8a0d0e0640 1 -- start start 2024-09-14T03:39:06.550 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.545+0000 7fe97dfbc640 1 -- start start 2024-09-14T03:39:06.551 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.545+0000 7f90945d4640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f908c09b640 0x7f908c08df10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:39:06.551 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.545+0000 7f90945d4640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f908c09ba20 0x7f908c08e450 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:39:06.551 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.545+0000 7f90945d4640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f908c12f6f0 0x7f908c08e990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:39:06.551 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.545+0000 7f9092370640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f908c09b640 0x7f908c08df10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:39:06.551 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.545+0000 7f90945d4640 1 -- --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7f908c0922a0 con 0x7f908c09b640 2024-09-14T03:39:06.552 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.545+0000 7f8a0d0e0640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f8a0808c780 0x7f8a0812ed60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:39:06.552 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.545+0000 7f8a0d0e0640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f8a0808d0a0 0x7f8a0812f2a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:39:06.552 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.545+0000 7f9092370640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f908c09b640 0x7f908c08df10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3300/0 says I am v2:172.21.15.83:36014/0 (socket says 172.21.15.83:36014) 2024-09-14T03:39:06.552 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.545+0000 7f9092370640 1 -- 172.21.15.83:0/1878529243 learned_addr learned my addr 172.21.15.83:0/1878529243 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:39:06.553 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.545+0000 7f90945d4640 1 -- 172.21.15.83:0/1878529243 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7f908c092120 con 0x7f908c09ba20 2024-09-14T03:39:06.553 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.545+0000 7f90945d4640 1 -- 172.21.15.83:0/1878529243 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7f908c092420 con 0x7f908c12f6f0 2024-09-14T03:39:06.553 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.545+0000 7f9092b71640 1 --2- 172.21.15.83:0/1878529243 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f908c12f6f0 0x7f908c08e990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:39:06.553 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.545+0000 7f9091b6f640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f908c09ba20 0x7f908c08e450 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:39:06.554 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.545+0000 7fe97dfbc640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fe960004800 0x7fe9781bf380 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:39:06.554 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.545+0000 7fe97dfbc640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fe978092880 0x7fe9781bf8c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:39:06.554 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.545+0000 7fe97dfbc640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fe9780931a0 0x7fe978127260 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:39:06.554 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.545+0000 7fe97dfbc640 1 -- --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7fe978093020 con 0x7fe960004800 2024-09-14T03:39:06.554 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.545+0000 7fe97dfbc640 1 -- --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7fe978092ea0 con 0x7fe9780931a0 2024-09-14T03:39:06.554 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.545+0000 7fe97dfbc640 1 -- --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7fe97812cdc0 con 0x7fe978092880 2024-09-14T03:39:06.554 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.545+0000 7fe9777fe640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fe960004800 0x7fe9781bf380 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:39:06.554 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.545+0000 7fe9777fe640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fe960004800 0x7fe9781bf380 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3300/0 says I am v2:172.21.15.83:36038/0 (socket says 172.21.15.83:36038) 2024-09-14T03:39:06.554 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.545+0000 7fe9777fe640 1 -- 172.21.15.83:0/2996718500 learned_addr learned my addr 172.21.15.83:0/2996718500 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:39:06.554 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.545+0000 7fe977fff640 1 --2- 172.21.15.83:0/2996718500 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fe9780931a0 0x7fe978127260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:39:06.555 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.546+0000 7fe976ffd640 1 --2- 172.21.15.83:0/2996718500 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fe978092880 0x7fe9781bf8c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:39:06.555 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.546+0000 7fe9777fe640 1 -- 172.21.15.83:0/2996718500 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fe978092880 msgr2=0x7fe9781bf8c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:06.555 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.546+0000 7fe9777fe640 1 --2- 172.21.15.83:0/2996718500 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fe978092880 0x7fe9781bf8c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:06.555 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.546+0000 7fe9777fe640 1 -- 172.21.15.83:0/2996718500 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fe9780931a0 msgr2=0x7fe978127260 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:06.555 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.546+0000 7fe9777fe640 1 --2- 172.21.15.83:0/2996718500 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fe9780931a0 0x7fe978127260 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:06.555 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.546+0000 7fe9777fe640 1 -- 172.21.15.83:0/2996718500 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fe9780701b0 con 0x7fe960004800 2024-09-14T03:39:06.555 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.545+0000 7f8a0d0e0640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f8a0812e3a0 0x7f8a0812f7e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:39:06.555 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.545+0000 7f8a0d0e0640 1 -- --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7f8a08136a60 con 0x7f8a0808c780 2024-09-14T03:39:06.555 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.545+0000 7f8a0d0e0640 1 -- --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7f8a081368e0 con 0x7f8a0808d0a0 2024-09-14T03:39:06.555 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.545+0000 7f8a06d9d640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f8a0808c780 0x7f8a0812ed60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:39:06.555 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.545+0000 7f8a0d0e0640 1 -- --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7f8a08136be0 con 0x7f8a0812e3a0 2024-09-14T03:39:06.556 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.545+0000 7f8a06d9d640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f8a0808c780 0x7f8a0812ed60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3300/0 says I am v2:172.21.15.83:36028/0 (socket says 172.21.15.83:36028) 2024-09-14T03:39:06.556 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.545+0000 7f8a06d9d640 1 -- 172.21.15.83:0/2803130687 learned_addr learned my addr 172.21.15.83:0/2803130687 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:39:06.556 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.545+0000 7f8a0759e640 1 --2- 172.21.15.83:0/2803130687 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f8a0812e3a0 0x7f8a0812f7e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:39:06.556 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.545+0000 7f8a0659c640 1 --2- 172.21.15.83:0/2803130687 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f8a0808d0a0 0x7f8a0812f2a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:39:06.556 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.546+0000 7f8a06d9d640 1 -- 172.21.15.83:0/2803130687 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f8a0812e3a0 msgr2=0x7f8a0812f7e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:06.556 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.546+0000 7f8a06d9d640 1 --2- 172.21.15.83:0/2803130687 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f8a0812e3a0 0x7f8a0812f7e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:06.556 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.546+0000 7f8a06d9d640 1 -- 172.21.15.83:0/2803130687 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f8a0808d0a0 msgr2=0x7f8a0812f2a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:06.556 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.546+0000 7f8a06d9d640 1 --2- 172.21.15.83:0/2803130687 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f8a0808d0a0 0x7f8a0812f2a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:06.556 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.546+0000 7f8a06d9d640 1 -- 172.21.15.83:0/2803130687 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f8a080922d0 con 0x7f8a0808c780 2024-09-14T03:39:06.556 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.546+0000 7f8a06d9d640 1 --2- 172.21.15.83:0/2803130687 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f8a0808c780 0x7f8a0812ed60 secure :-1 s=READY pgs=79 cs=0 l=1 rev1=1 crypto rx=0x7f89f800e790 tx=0x7f89f800ec50 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:39:06.557 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.545+0000 7f9092370640 1 -- 172.21.15.83:0/1878529243 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f908c12f6f0 msgr2=0x7f908c08e990 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:06.557 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.545+0000 7f9092370640 1 --2- 172.21.15.83:0/1878529243 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f908c12f6f0 0x7f908c08e990 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:06.557 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.546+0000 7f9092370640 1 -- 172.21.15.83:0/1878529243 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f908c09ba20 msgr2=0x7f908c08e450 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:06.557 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.546+0000 7f9092370640 1 --2- 172.21.15.83:0/1878529243 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f908c09ba20 0x7f908c08e450 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:06.557 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.546+0000 7f9092370640 1 -- 172.21.15.83:0/1878529243 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f908c08eed0 con 0x7f908c09b640 2024-09-14T03:39:06.557 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.546+0000 7f9092370640 1 --2- 172.21.15.83:0/1878529243 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f908c09b640 0x7f908c08df10 secure :-1 s=READY pgs=78 cs=0 l=1 rev1=1 crypto rx=0x7f908800e9c0 tx=0x7f908800ee80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:39:06.558 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.546+0000 7f9092b71640 1 --2- 172.21.15.83:0/1878529243 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f908c12f6f0 0x7f908c08e990 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:39:06.558 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.546+0000 7f9091b6f640 1 --2- 172.21.15.83:0/1878529243 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f908c09ba20 0x7f908c08e450 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-09-14T03:39:06.558 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.546+0000 7fe9777fe640 1 --2- 172.21.15.83:0/2996718500 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fe960004800 0x7fe9781bf380 secure :-1 s=READY pgs=80 cs=0 l=1 rev1=1 crypto rx=0x7fe968005f60 tx=0x7fe96800f790 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:39:06.559 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.546+0000 7f9082ffd640 1 -- 172.21.15.83:0/1878529243 <== mon.0 v2:172.21.15.83:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f908800cc60 con 0x7f908c09b640 2024-09-14T03:39:06.559 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.546+0000 7f9082ffd640 1 -- 172.21.15.83:0/1878529243 <== mon.0 v2:172.21.15.83:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f908800ce00 con 0x7f908c09b640 2024-09-14T03:39:06.559 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.546+0000 7f89ef7fe640 1 -- 172.21.15.83:0/2803130687 <== mon.0 v2:172.21.15.83:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f89f800eea0 con 0x7f8a0808c780 2024-09-14T03:39:06.559 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.546+0000 7f89ef7fe640 1 -- 172.21.15.83:0/2803130687 <== mon.0 v2:172.21.15.83:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f89f800cdb0 con 0x7f8a0808c780 2024-09-14T03:39:06.559 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.546+0000 7f89ef7fe640 1 -- 172.21.15.83:0/2803130687 <== mon.0 v2:172.21.15.83:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f89f8010690 con 0x7f8a0808c780 2024-09-14T03:39:06.559 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.546+0000 7f8a0d0e0640 1 -- 172.21.15.83:0/2803130687 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f8a0812fd20 con 0x7f8a0808c780 2024-09-14T03:39:06.560 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.546+0000 7f8a0d0e0640 1 -- 172.21.15.83:0/2803130687 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f8a08130710 con 0x7f8a0808c780 2024-09-14T03:39:06.560 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.546+0000 7f9082ffd640 1 -- 172.21.15.83:0/1878529243 <== mon.0 v2:172.21.15.83:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f908800cc60 con 0x7f908c09b640 2024-09-14T03:39:06.560 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.546+0000 7f90945d4640 1 -- 172.21.15.83:0/1878529243 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f908c06d6f0 con 0x7f908c09b640 2024-09-14T03:39:06.560 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.546+0000 7f90945d4640 1 -- 172.21.15.83:0/1878529243 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f908c1ce600 con 0x7f908c09b640 2024-09-14T03:39:06.560 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.546+0000 7fe95ffff640 1 -- 172.21.15.83:0/2996718500 <== mon.0 v2:172.21.15.83:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe96801e070 con 0x7fe960004800 2024-09-14T03:39:06.561 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.547+0000 7fe97dfbc640 1 -- 172.21.15.83:0/2996718500 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fe978126400 con 0x7fe960004800 2024-09-14T03:39:06.561 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.547+0000 7fe97dfbc640 1 -- 172.21.15.83:0/2996718500 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fe9781bfe00 con 0x7fe960004800 2024-09-14T03:39:06.561 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.547+0000 7f90945d4640 1 -- 172.21.15.83:0/1878529243 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f908c098930 con 0x7f908c09b640 2024-09-14T03:39:06.561 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.548+0000 7f8a0d0e0640 1 -- 172.21.15.83:0/2803130687 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f8a0808cb60 con 0x7f8a0808c780 2024-09-14T03:39:06.562 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.548+0000 7fe95ffff640 1 -- 172.21.15.83:0/2996718500 <== mon.0 v2:172.21.15.83:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fe968009720 con 0x7fe960004800 2024-09-14T03:39:06.562 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.548+0000 7fe97dfbc640 1 -- 172.21.15.83:0/2996718500 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fe97812bac0 con 0x7fe960004800 2024-09-14T03:39:06.562 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.548+0000 7fe95ffff640 1 -- 172.21.15.83:0/2996718500 <== mon.0 v2:172.21.15.83:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe968028060 con 0x7fe960004800 2024-09-14T03:39:06.562 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.548+0000 7f9082ffd640 1 -- 172.21.15.83:0/1878529243 <== mon.0 v2:172.21.15.83:3300/0 4 ==== mgrmap(e 5) ==== 55999+0+0 (secure 0 0 0) 0x7f9088005020 con 0x7f908c09b640 2024-09-14T03:39:06.562 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.548+0000 7f9082ffd640 1 -- 172.21.15.83:0/1878529243 <== mon.0 v2:172.21.15.83:3300/0 5 ==== osd_map(18..18 src has 1..18) ==== 2868+0+0 (secure 0 0 0) 0x7f908805de60 con 0x7f908c09b640 2024-09-14T03:39:06.563 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.549+0000 7f89ef7fe640 1 -- 172.21.15.83:0/2803130687 <== mon.0 v2:172.21.15.83:3300/0 4 ==== mgrmap(e 5) ==== 55999+0+0 (secure 0 0 0) 0x7f89f8010830 con 0x7f8a0808c780 2024-09-14T03:39:06.563 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.549+0000 7fe95ffff640 1 -- 172.21.15.83:0/2996718500 <== mon.0 v2:172.21.15.83:3300/0 4 ==== mgrmap(e 5) ==== 55999+0+0 (secure 0 0 0) 0x7fe968021070 con 0x7fe960004800 2024-09-14T03:39:06.563 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.549+0000 7f89ef7fe640 1 -- 172.21.15.83:0/2803130687 <== mon.0 v2:172.21.15.83:3300/0 5 ==== osd_map(18..18 src has 1..18) ==== 2868+0+0 (secure 0 0 0) 0x7f89f805e410 con 0x7f8a0808c780 2024-09-14T03:39:06.563 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.549+0000 7fe95ffff640 1 -- 172.21.15.83:0/2996718500 <== mon.0 v2:172.21.15.83:3300/0 5 ==== osd_map(18..18 src has 1..18) ==== 2868+0+0 (secure 0 0 0) 0x7fe96805ef70 con 0x7fe960004800 2024-09-14T03:39:06.564 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.553+0000 7f9082ffd640 1 -- 172.21.15.83:0/1878529243 <== mon.0 v2:172.21.15.83:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+151260 (secure 0 0 0) 0x7f908805cfb0 con 0x7f908c09b640 2024-09-14T03:39:06.564 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.556+0000 7fe95ffff640 1 -- 172.21.15.83:0/2996718500 <== mon.0 v2:172.21.15.83:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+151260 (secure 0 0 0) 0x7fe978126400 con 0x7fe960004800 2024-09-14T03:39:06.564 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.560+0000 7f89ef7fe640 1 -- 172.21.15.83:0/2803130687 <== mon.0 v2:172.21.15.83:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+151260 (secure 0 0 0) 0x7f89f803d680 con 0x7f8a0808c780 2024-09-14T03:39:06.665 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.663+0000 7f4c2e383640 1 -- 172.21.15.83:0/1851606781 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 0} v 0) -- 0x7f4c2812c6e0 con 0x7f4c28096400 2024-09-14T03:39:06.665 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.664+0000 7f4c24ff9640 1 -- 172.21.15.83:0/1851606781 <== mon.2 v2:172.21.15.83:3302/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 0}]=0 v0) ==== 74+0+12 (secure 0 0 0) 0x7f4c2003d320 con 0x7f4c28096400 2024-09-14T03:39:06.666 INFO:teuthology.orchestra.run.smithi083.stdout:38654705667 2024-09-14T03:39:06.668 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.666+0000 7f4c2e383640 1 -- 172.21.15.83:0/1851606781 >> v1:172.21.15.83:6800/1934165865 conn(0x7f4c14048000 legacy=0x7f4c1404a4c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:06.668 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.666+0000 7f4c2e383640 1 -- 172.21.15.83:0/1851606781 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f4c28096400 msgr2=0x7f4c2812ab70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:06.668 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.666+0000 7f4c2e383640 1 --2- 172.21.15.83:0/1851606781 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f4c28096400 0x7f4c2812ab70 secure :-1 s=READY pgs=46 cs=0 l=1 rev1=1 crypto rx=0x7f4c20009fd0 tx=0x7f4c2000c800 comp rx=0 tx=0).stop 2024-09-14T03:39:06.668 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.666+0000 7f4c2e383640 1 -- 172.21.15.83:0/1851606781 shutdown_connections 2024-09-14T03:39:06.668 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.666+0000 7f4c2e383640 1 --2- 172.21.15.83:0/1851606781 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f4c28096400 0x7f4c2812ab70 unknown :-1 s=CLOSED pgs=46 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:06.668 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.666+0000 7f4c2e383640 1 --2- 172.21.15.83:0/1851606781 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f4c2806ede0 0x7f4c281c43c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:06.669 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.666+0000 7f4c2e383640 1 --2- 172.21.15.83:0/1851606781 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f4c2806e4a0 0x7f4c281c3e80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:06.669 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.666+0000 7f4c2e383640 1 -- 172.21.15.83:0/1851606781 >> 172.21.15.83:0/1851606781 conn(0x7f4c2809cdf0 msgr2=0x7f4c281304c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:39:06.669 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.667+0000 7f4c2e383640 1 -- 172.21.15.83:0/1851606781 shutdown_connections 2024-09-14T03:39:06.669 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.667+0000 7f4c2e383640 1 -- 172.21.15.83:0/1851606781 wait complete. 2024-09-14T03:39:06.683 INFO:tasks.ceph.ceph_manager.ceph:need seq 38654705668 got 38654705667 for osd.0 2024-09-14T03:39:06.712 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.711+0000 7fe97dfbc640 1 -- 172.21.15.83:0/2996718500 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 3} v 0) -- 0x7fe978073ee0 con 0x7fe960004800 2024-09-14T03:39:06.712 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.711+0000 7fe95ffff640 1 -- 172.21.15.83:0/2996718500 <== mon.0 v2:172.21.15.83:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 3}]=0 v0) ==== 74+0+12 (secure 0 0 0) 0x7fe9780701b0 con 0x7fe960004800 2024-09-14T03:39:06.713 INFO:teuthology.orchestra.run.smithi083.stdout:47244640259 2024-09-14T03:39:06.714 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.713+0000 7fe97dfbc640 1 -- 172.21.15.83:0/2996718500 >> v1:172.21.15.83:6800/1934165865 conn(0x7fe964047fd0 legacy=0x7fe96404a490 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:06.715 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.713+0000 7fe97dfbc640 1 -- 172.21.15.83:0/2996718500 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fe960004800 msgr2=0x7fe9781bf380 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:06.715 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.713+0000 7fe97dfbc640 1 --2- 172.21.15.83:0/2996718500 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fe960004800 0x7fe9781bf380 secure :-1 s=READY pgs=80 cs=0 l=1 rev1=1 crypto rx=0x7fe968005f60 tx=0x7fe96800f790 comp rx=0 tx=0).stop 2024-09-14T03:39:06.715 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.713+0000 7fe97dfbc640 1 -- 172.21.15.83:0/2996718500 shutdown_connections 2024-09-14T03:39:06.715 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.713+0000 7fe97dfbc640 1 --2- 172.21.15.83:0/2996718500 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fe9780931a0 0x7fe978127260 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:06.715 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.713+0000 7fe97dfbc640 1 --2- 172.21.15.83:0/2996718500 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fe978092880 0x7fe9781bf8c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:06.715 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.713+0000 7fe97dfbc640 1 --2- 172.21.15.83:0/2996718500 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fe960004800 0x7fe9781bf380 unknown :-1 s=CLOSED pgs=80 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:06.715 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.713+0000 7fe97dfbc640 1 -- 172.21.15.83:0/2996718500 >> 172.21.15.83:0/2996718500 conn(0x7fe97809c630 msgr2=0x7fe9781233d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:39:06.715 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.713+0000 7fe97dfbc640 1 -- 172.21.15.83:0/2996718500 shutdown_connections 2024-09-14T03:39:06.715 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.713+0000 7fe97dfbc640 1 -- 172.21.15.83:0/2996718500 wait complete. 2024-09-14T03:39:06.725 INFO:tasks.ceph.ceph_manager.ceph:need seq 47244640260 got 47244640259 for osd.3 2024-09-14T03:39:06.729 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.728+0000 7f90945d4640 1 -- 172.21.15.83:0/1878529243 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 1} v 0) -- 0x7f908c12a4a0 con 0x7f908c09b640 2024-09-14T03:39:06.730 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.729+0000 7f9082ffd640 1 -- 172.21.15.83:0/1878529243 <== mon.0 v2:172.21.15.83:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 1}]=0 v0) ==== 74+0+12 (secure 0 0 0) 0x7f908803d070 con 0x7f908c09b640 2024-09-14T03:39:06.730 INFO:teuthology.orchestra.run.smithi083.stdout:47244640259 2024-09-14T03:39:06.732 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.730+0000 7f90945d4640 1 -- 172.21.15.83:0/1878529243 >> v1:172.21.15.83:6800/1934165865 conn(0x7f90700481d0 legacy=0x7f907004a690 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:06.732 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.730+0000 7f90945d4640 1 -- 172.21.15.83:0/1878529243 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f908c09b640 msgr2=0x7f908c08df10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:06.732 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.730+0000 7f90945d4640 1 --2- 172.21.15.83:0/1878529243 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f908c09b640 0x7f908c08df10 secure :-1 s=READY pgs=78 cs=0 l=1 rev1=1 crypto rx=0x7f908800e9c0 tx=0x7f908800ee80 comp rx=0 tx=0).stop 2024-09-14T03:39:06.732 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.730+0000 7f90945d4640 1 -- 172.21.15.83:0/1878529243 shutdown_connections 2024-09-14T03:39:06.732 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.730+0000 7f90945d4640 1 --2- 172.21.15.83:0/1878529243 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f908c12f6f0 0x7f908c08e990 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:06.732 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.730+0000 7f90945d4640 1 --2- 172.21.15.83:0/1878529243 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f908c09ba20 0x7f908c08e450 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:06.732 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.730+0000 7f90945d4640 1 --2- 172.21.15.83:0/1878529243 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f908c09b640 0x7f908c08df10 unknown :-1 s=CLOSED pgs=78 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:06.732 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.730+0000 7f90945d4640 1 -- 172.21.15.83:0/1878529243 >> 172.21.15.83:0/1878529243 conn(0x7f908c09a610 msgr2=0x7f908c0974a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:39:06.733 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.730+0000 7f90945d4640 1 -- 172.21.15.83:0/1878529243 shutdown_connections 2024-09-14T03:39:06.733 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.731+0000 7f90945d4640 1 -- 172.21.15.83:0/1878529243 wait complete. 2024-09-14T03:39:06.737 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.736+0000 7f8a0d0e0640 1 -- 172.21.15.83:0/2803130687 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 2} v 0) -- 0x7f8a08073ee0 con 0x7f8a0808c780 2024-09-14T03:39:06.738 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.736+0000 7f89ef7fe640 1 -- 172.21.15.83:0/2803130687 <== mon.0 v2:172.21.15.83:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 2}]=0 v0) ==== 74+0+12 (secure 0 0 0) 0x7f89f803d8e0 con 0x7f8a0808c780 2024-09-14T03:39:06.738 INFO:teuthology.orchestra.run.smithi083.stdout:47244640259 2024-09-14T03:39:06.740 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.738+0000 7f8a0d0e0640 1 -- 172.21.15.83:0/2803130687 >> v1:172.21.15.83:6800/1934165865 conn(0x7f89f40480a0 legacy=0x7f89f404a560 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:06.740 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.738+0000 7f8a0d0e0640 1 -- 172.21.15.83:0/2803130687 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f8a0808c780 msgr2=0x7f8a0812ed60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:06.740 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.738+0000 7f8a0d0e0640 1 --2- 172.21.15.83:0/2803130687 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f8a0808c780 0x7f8a0812ed60 secure :-1 s=READY pgs=79 cs=0 l=1 rev1=1 crypto rx=0x7f89f800e790 tx=0x7f89f800ec50 comp rx=0 tx=0).stop 2024-09-14T03:39:06.740 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.738+0000 7f8a0d0e0640 1 -- 172.21.15.83:0/2803130687 shutdown_connections 2024-09-14T03:39:06.740 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.738+0000 7f8a0d0e0640 1 --2- 172.21.15.83:0/2803130687 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f8a0812e3a0 0x7f8a0812f7e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:06.740 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.738+0000 7f8a0d0e0640 1 --2- 172.21.15.83:0/2803130687 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f8a0808d0a0 0x7f8a0812f2a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:06.740 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.738+0000 7f8a0d0e0640 1 --2- 172.21.15.83:0/2803130687 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f8a0808c780 0x7f8a0812ed60 unknown :-1 s=CLOSED pgs=79 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:06.741 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.738+0000 7f8a0d0e0640 1 -- 172.21.15.83:0/2803130687 >> 172.21.15.83:0/2803130687 conn(0x7f8a080a0c80 msgr2=0x7f8a0806dc10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:39:06.741 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.738+0000 7f8a0d0e0640 1 -- 172.21.15.83:0/2803130687 shutdown_connections 2024-09-14T03:39:06.741 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:06.738+0000 7f8a0d0e0640 1 -- 172.21.15.83:0/2803130687 wait complete. 2024-09-14T03:39:06.742 INFO:tasks.ceph.ceph_manager.ceph:need seq 47244640260 got 47244640259 for osd.1 2024-09-14T03:39:06.750 INFO:tasks.ceph.ceph_manager.ceph:need seq 47244640260 got 47244640259 for osd.2 2024-09-14T03:39:07.684 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.0 2024-09-14T03:39:07.726 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.3 2024-09-14T03:39:07.743 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.1 2024-09-14T03:39:07.751 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.2 2024-09-14T03:39:07.801 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.800+0000 7ff6e892c640 1 -- 172.21.15.83:0/3262092646 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7ff6e412e720 msgr2=0x7ff6e4096800 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:07.802 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.800+0000 7ff6e892c640 1 --2- 172.21.15.83:0/3262092646 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7ff6e412e720 0x7ff6e4096800 secure :-1 s=READY pgs=47 cs=0 l=1 rev1=1 crypto rx=0x7ff6d8009900 tx=0x7ff6d802a490 comp rx=0 tx=0).stop 2024-09-14T03:39:07.802 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.800+0000 7ff6e892c640 1 -- 172.21.15.83:0/3262092646 shutdown_connections 2024-09-14T03:39:07.802 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.800+0000 7ff6e892c640 1 --2- 172.21.15.83:0/3262092646 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7ff6e412e720 0x7ff6e4096800 unknown :-1 s=CLOSED pgs=47 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:07.802 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.800+0000 7ff6e892c640 1 --2- 172.21.15.83:0/3262092646 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7ff6e409b500 0x7ff6e412e1e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:07.802 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.800+0000 7ff6e892c640 1 -- 172.21.15.83:0/3262092646 >> 172.21.15.83:0/3262092646 conn(0x7ff6e409e920 msgr2=0x7ff6e409adf0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:39:07.802 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.800+0000 7ff6e892c640 1 -- 172.21.15.83:0/3262092646 shutdown_connections 2024-09-14T03:39:07.802 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.800+0000 7ff6e892c640 1 -- 172.21.15.83:0/3262092646 wait complete. 2024-09-14T03:39:07.803 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.801+0000 7ff6e892c640 1 Processor -- start 2024-09-14T03:39:07.803 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.801+0000 7ff6e892c640 1 -- start start 2024-09-14T03:39:07.804 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.802+0000 7ff6e892c640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7ff6e4096d40 0x7ff6e4123a90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:39:07.804 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.802+0000 7ff6e892c640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7ff6e409b500 0x7ff6e4123fd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:39:07.804 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.802+0000 7ff6e892c640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7ff6e412e720 0x7ff6e41270d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:39:07.804 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.802+0000 7ff6e892c640 1 -- --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7ff6e4136ba0 con 0x7ff6e409b500 2024-09-14T03:39:07.804 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.802+0000 7ff6e892c640 1 -- --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7ff6e4136a20 con 0x7ff6e4096d40 2024-09-14T03:39:07.804 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.802+0000 7ff6e892c640 1 -- --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7ff6e4136d20 con 0x7ff6e412e720 2024-09-14T03:39:07.804 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.802+0000 7ff6e1d9b640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7ff6e409b500 0x7ff6e4123fd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:39:07.804 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.802+0000 7ff6e2d9d640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7ff6e412e720 0x7ff6e41270d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:39:07.804 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.802+0000 7ff6e2d9d640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7ff6e412e720 0x7ff6e41270d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3302/0 says I am v2:172.21.15.83:33020/0 (socket says 172.21.15.83:33020) 2024-09-14T03:39:07.804 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.802+0000 7ff6e1d9b640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7ff6e409b500 0x7ff6e4123fd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3300/0 says I am v2:172.21.15.83:36054/0 (socket says 172.21.15.83:36054) 2024-09-14T03:39:07.804 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.802+0000 7ff6e2d9d640 1 -- 172.21.15.83:0/3840682549 learned_addr learned my addr 172.21.15.83:0/3840682549 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:39:07.805 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.802+0000 7ff6e259c640 1 --2- 172.21.15.83:0/3840682549 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7ff6e4096d40 0x7ff6e4123a90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:39:07.806 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.803+0000 7ff6e1d9b640 1 -- 172.21.15.83:0/3840682549 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7ff6e412e720 msgr2=0x7ff6e41270d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:07.806 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.803+0000 7ff6e1d9b640 1 --2- 172.21.15.83:0/3840682549 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7ff6e412e720 0x7ff6e41270d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:07.806 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.803+0000 7ff6e1d9b640 1 -- 172.21.15.83:0/3840682549 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7ff6e4096d40 msgr2=0x7ff6e4123a90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:07.806 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.803+0000 7ff6e1d9b640 1 --2- 172.21.15.83:0/3840682549 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7ff6e4096d40 0x7ff6e4123a90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:07.806 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.803+0000 7ff6e1d9b640 1 -- 172.21.15.83:0/3840682549 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7ff6e4125180 con 0x7ff6e409b500 2024-09-14T03:39:07.806 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.803+0000 7ff6e2d9d640 1 --2- 172.21.15.83:0/3840682549 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7ff6e412e720 0x7ff6e41270d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:39:07.806 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.803+0000 7ff6e259c640 1 --2- 172.21.15.83:0/3840682549 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7ff6e4096d40 0x7ff6e4123a90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:39:07.806 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.803+0000 7ff6e1d9b640 1 --2- 172.21.15.83:0/3840682549 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7ff6e409b500 0x7ff6e4123fd0 secure :-1 s=READY pgs=82 cs=0 l=1 rev1=1 crypto rx=0x7ff6d8031000 tx=0x7ff6d80038a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:39:07.807 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.803+0000 7ff6d2ffd640 1 -- 172.21.15.83:0/3840682549 <== mon.0 v2:172.21.15.83:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff6d803f070 con 0x7ff6e409b500 2024-09-14T03:39:07.807 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.803+0000 7ff6e892c640 1 -- 172.21.15.83:0/3840682549 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7ff6e41262d0 con 0x7ff6e409b500 2024-09-14T03:39:07.807 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.804+0000 7ff6d2ffd640 1 -- 172.21.15.83:0/3840682549 <== mon.0 v2:172.21.15.83:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7ff6d8004890 con 0x7ff6e409b500 2024-09-14T03:39:07.807 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.804+0000 7ff6d2ffd640 1 -- 172.21.15.83:0/3840682549 <== mon.0 v2:172.21.15.83:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff6d8035030 con 0x7ff6e409b500 2024-09-14T03:39:07.807 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.804+0000 7ff6e892c640 1 -- 172.21.15.83:0/3840682549 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7ff6e4124510 con 0x7ff6e409b500 2024-09-14T03:39:07.807 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.805+0000 7ff6d2ffd640 1 -- 172.21.15.83:0/3840682549 <== mon.0 v2:172.21.15.83:3300/0 4 ==== mgrmap(e 5) ==== 55999+0+0 (secure 0 0 0) 0x7ff6d8042070 con 0x7ff6e409b500 2024-09-14T03:39:07.808 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.805+0000 7ff6e892c640 1 -- 172.21.15.83:0/3840682549 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7ff6e412e1e0 con 0x7ff6e409b500 2024-09-14T03:39:07.808 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.805+0000 7ff6d2ffd640 1 -- 172.21.15.83:0/3840682549 <== mon.0 v2:172.21.15.83:3300/0 5 ==== osd_map(18..18 src has 1..18) ==== 2868+0+0 (secure 0 0 0) 0x7ff6d807e720 con 0x7ff6e409b500 2024-09-14T03:39:07.814 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.813+0000 7ff6d2ffd640 1 -- 172.21.15.83:0/3840682549 <== mon.0 v2:172.21.15.83:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+151260 (secure 0 0 0) 0x7ff6d807d870 con 0x7ff6e409b500 2024-09-14T03:39:07.846 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.843+0000 7f1cdec67640 1 -- 172.21.15.83:0/2865729649 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f1cd809c8f0 msgr2=0x7f1cd8122760 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:07.846 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.843+0000 7f1cdec67640 1 --2- 172.21.15.83:0/2865729649 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f1cd809c8f0 0x7f1cd8122760 secure :-1 s=READY pgs=43 cs=0 l=1 rev1=1 crypto rx=0x7f1cd0005fd0 tx=0x7f1cd002a6e0 comp rx=0 tx=0).stop 2024-09-14T03:39:07.846 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.844+0000 7f1cdec67640 1 -- 172.21.15.83:0/2865729649 shutdown_connections 2024-09-14T03:39:07.846 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.844+0000 7f1cdec67640 1 --2- 172.21.15.83:0/2865729649 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f1cd809c8f0 0x7f1cd8122760 unknown :-1 s=CLOSED pgs=43 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:07.846 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.844+0000 7f1cdec67640 1 --2- 172.21.15.83:0/2865729649 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f1cd812c8f0 0x7f1cd8122220 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:07.846 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.844+0000 7f1cdec67640 1 --2- 172.21.15.83:0/2865729649 >> v2:172.21.15.83:3300/0 conn(0x7f1cd812bfd0 0x7f1cd812c3b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:07.846 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.844+0000 7f1cdec67640 1 -- 172.21.15.83:0/2865729649 >> 172.21.15.83:0/2865729649 conn(0x7f1cd809b660 msgr2=0x7f1cd809ba70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:39:07.846 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.844+0000 7f1cdec67640 1 -- 172.21.15.83:0/2865729649 shutdown_connections 2024-09-14T03:39:07.847 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.844+0000 7f1cdec67640 1 -- 172.21.15.83:0/2865729649 wait complete. 2024-09-14T03:39:07.847 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.844+0000 7f1cdec67640 1 Processor -- start 2024-09-14T03:39:07.847 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.845+0000 7f1cdec67640 1 -- start start 2024-09-14T03:39:07.848 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.845+0000 7f1cdec67640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f1cd809c8f0 0x7f1cd806e960 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:39:07.848 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.845+0000 7f1cdec67640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f1cd812bfd0 0x7f1cd806eea0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:39:07.849 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.845+0000 7f1cdec67640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f1cd812c8f0 0x7f1cd80721b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:39:07.849 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.845+0000 7f1cdec67640 1 -- --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7f1cd8133700 con 0x7f1cd812bfd0 2024-09-14T03:39:07.849 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.845+0000 7f1cdec67640 1 -- --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7f1cd8133580 con 0x7f1cd812c8f0 2024-09-14T03:39:07.849 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.845+0000 7f1cdec67640 1 -- --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7f1cd8133880 con 0x7f1cd809c8f0 2024-09-14T03:39:07.849 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.845+0000 7f1cdd204640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f1cd812c8f0 0x7f1cd80721b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:39:07.849 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.845+0000 7f1cdd204640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f1cd812c8f0 0x7f1cd80721b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3301/0 says I am v2:172.21.15.83:60720/0 (socket says 172.21.15.83:60720) 2024-09-14T03:39:07.849 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.845+0000 7f1cdd204640 1 -- 172.21.15.83:0/1528282311 learned_addr learned my addr 172.21.15.83:0/1528282311 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:39:07.849 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.845+0000 7f1cdca03640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f1cd809c8f0 0x7f1cd806e960 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:39:07.849 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.845+0000 7f1cd7fff640 1 --2- 172.21.15.83:0/1528282311 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f1cd812bfd0 0x7f1cd806eea0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:39:07.849 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.846+0000 7f1cdd204640 1 -- 172.21.15.83:0/1528282311 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f1cd809c8f0 msgr2=0x7f1cd806e960 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:07.849 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.846+0000 7f1cdd204640 1 --2- 172.21.15.83:0/1528282311 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f1cd809c8f0 0x7f1cd806e960 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:07.849 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.846+0000 7f1cdd204640 1 -- 172.21.15.83:0/1528282311 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f1cd812bfd0 msgr2=0x7f1cd806eea0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:07.849 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.846+0000 7f1cdd204640 1 --2- 172.21.15.83:0/1528282311 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f1cd812bfd0 0x7f1cd806eea0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:07.850 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.846+0000 7f1cdd204640 1 -- 172.21.15.83:0/1528282311 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f1cd8071ef0 con 0x7f1cd812c8f0 2024-09-14T03:39:07.850 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.846+0000 7f1cdca03640 1 --2- 172.21.15.83:0/1528282311 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f1cd809c8f0 0x7f1cd806e960 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:39:07.850 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.846+0000 7f1cd7fff640 1 --2- 172.21.15.83:0/1528282311 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f1cd812bfd0 0x7f1cd806eea0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:39:07.850 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.846+0000 7f1cdd204640 1 --2- 172.21.15.83:0/1528282311 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f1cd812c8f0 0x7f1cd80721b0 secure :-1 s=READY pgs=44 cs=0 l=1 rev1=1 crypto rx=0x7f1cd0034040 tx=0x7f1cd0003e60 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:39:07.850 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.846+0000 7f1cd57fa640 1 -- 172.21.15.83:0/1528282311 <== mon.1 v2:172.21.15.83:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1cd0041020 con 0x7f1cd812c8f0 2024-09-14T03:39:07.850 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.846+0000 7f1cd57fa640 1 -- 172.21.15.83:0/1528282311 <== mon.1 v2:172.21.15.83:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f1cd0004040 con 0x7f1cd812c8f0 2024-09-14T03:39:07.850 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.846+0000 7f1cd57fa640 1 -- 172.21.15.83:0/1528282311 <== mon.1 v2:172.21.15.83:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1cd0036040 con 0x7f1cd812c8f0 2024-09-14T03:39:07.850 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.846+0000 7f1cdec67640 1 -- 172.21.15.83:0/1528282311 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f1cd8070b40 con 0x7f1cd812c8f0 2024-09-14T03:39:07.850 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.846+0000 7f1cdec67640 1 -- 172.21.15.83:0/1528282311 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7f1cd80726f0 con 0x7f1cd812c8f0 2024-09-14T03:39:07.851 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.848+0000 7f1cdec67640 1 -- 172.21.15.83:0/1528282311 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f1cd8073a00 con 0x7f1cd812c8f0 2024-09-14T03:39:07.851 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.848+0000 7f1cd57fa640 1 -- 172.21.15.83:0/1528282311 <== mon.1 v2:172.21.15.83:3301/0 4 ==== mgrmap(e 5) ==== 55999+0+0 (secure 0 0 0) 0x7f1cd0044020 con 0x7f1cd812c8f0 2024-09-14T03:39:07.851 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.848+0000 7f1cd57fa640 1 -- 172.21.15.83:0/1528282311 <== mon.1 v2:172.21.15.83:3301/0 5 ==== osd_map(18..18 src has 1..18) ==== 2868+0+0 (secure 0 0 0) 0x7f1cd007e7b0 con 0x7f1cd812c8f0 2024-09-14T03:39:07.853 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.852+0000 7f1cd57fa640 1 -- 172.21.15.83:0/1528282311 <== mon.1 v2:172.21.15.83:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+151260 (secure 0 0 0) 0x7f1cd8073a00 con 0x7f1cd812c8f0 2024-09-14T03:39:07.883 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.881+0000 7feda99fd640 1 -- 172.21.15.83:0/1166353669 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7feda408c780 msgr2=0x7feda408cb60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:07.883 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.881+0000 7feda99fd640 1 --2- 172.21.15.83:0/1166353669 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7feda408c780 0x7feda408cb60 secure :-1 s=READY pgs=45 cs=0 l=1 rev1=1 crypto rx=0x7fed94009860 tx=0x7fed9402a3d0 comp rx=0 tx=0).stop 2024-09-14T03:39:07.883 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.882+0000 7feda99fd640 1 -- 172.21.15.83:0/1166353669 shutdown_connections 2024-09-14T03:39:07.883 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.882+0000 7feda99fd640 1 --2- 172.21.15.83:0/1166353669 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7feda408d0a0 0x7feda412dc90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:07.883 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.882+0000 7feda99fd640 1 --2- 172.21.15.83:0/1166353669 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7feda408c780 0x7feda408cb60 unknown :-1 s=CLOSED pgs=45 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:07.884 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.882+0000 7feda99fd640 1 -- 172.21.15.83:0/1166353669 >> 172.21.15.83:0/1166353669 conn(0x7feda40a0c80 msgr2=0x7feda408b000 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:39:07.884 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.882+0000 7f26bf5ae640 1 -- 172.21.15.83:0/487399086 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f26b806bbc0 msgr2=0x7f26b806bfa0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:07.884 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.882+0000 7f26bf5ae640 1 --2- 172.21.15.83:0/487399086 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f26b806bbc0 0x7f26b806bfa0 secure :-1 s=READY pgs=46 cs=0 l=1 rev1=1 crypto rx=0x7f26b4009900 tx=0x7f26b402a4a0 comp rx=0 tx=0).stop 2024-09-14T03:39:07.884 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.882+0000 7feda99fd640 1 -- 172.21.15.83:0/1166353669 shutdown_connections 2024-09-14T03:39:07.885 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.882+0000 7f26bf5ae640 1 -- 172.21.15.83:0/487399086 shutdown_connections 2024-09-14T03:39:07.885 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.882+0000 7f26bf5ae640 1 --2- 172.21.15.83:0/487399086 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f26b8073810 0x7f26b806c4e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:07.885 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.882+0000 7feda99fd640 1 -- 172.21.15.83:0/1166353669 wait complete. 2024-09-14T03:39:07.885 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.882+0000 7f26bf5ae640 1 --2- 172.21.15.83:0/487399086 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f26b806bbc0 0x7f26b806bfa0 unknown :-1 s=CLOSED pgs=46 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:07.886 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.882+0000 7f26bf5ae640 1 -- 172.21.15.83:0/487399086 >> 172.21.15.83:0/487399086 conn(0x7f26b809d510 msgr2=0x7f26b809d920 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:39:07.886 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.882+0000 7f26bf5ae640 1 -- 172.21.15.83:0/487399086 shutdown_connections 2024-09-14T03:39:07.886 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.882+0000 7f26bf5ae640 1 -- 172.21.15.83:0/487399086 wait complete. 2024-09-14T03:39:07.886 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.882+0000 7feda99fd640 1 Processor -- start 2024-09-14T03:39:07.886 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.883+0000 7feda99fd640 1 -- start start 2024-09-14T03:39:07.886 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.883+0000 7f26bf5ae640 1 Processor -- start 2024-09-14T03:39:07.887 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.883+0000 7f26bf5ae640 1 -- start start 2024-09-14T03:39:07.887 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.883+0000 7feda99fd640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7feda408c780 0x7feda412ed60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:39:07.887 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.883+0000 7feda99fd640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7feda408d0a0 0x7feda412f2a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:39:07.887 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.883+0000 7feda99fd640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7feda412e1d0 0x7feda412f7e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:39:07.887 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.883+0000 7feda99fd640 1 -- --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7feda4136ac0 con 0x7feda408d0a0 2024-09-14T03:39:07.888 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.883+0000 7feda99fd640 1 -- --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7feda4136940 con 0x7feda412e1d0 2024-09-14T03:39:07.888 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.883+0000 7feda99fd640 1 -- --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7feda4136c40 con 0x7feda408c780 2024-09-14T03:39:07.888 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.883+0000 7feda2ffd640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7feda408c780 0x7feda412ed60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:39:07.888 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.883+0000 7feda37fe640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7feda412e1d0 0x7feda412f7e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:39:07.888 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.883+0000 7feda37fe640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7feda412e1d0 0x7feda412f7e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3301/0 says I am v2:172.21.15.83:60742/0 (socket says 172.21.15.83:60742) 2024-09-14T03:39:07.888 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.883+0000 7feda37fe640 1 -- 172.21.15.83:0/3272817285 learned_addr learned my addr 172.21.15.83:0/3272817285 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:39:07.888 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.883+0000 7feda27fc640 1 --2- 172.21.15.83:0/3272817285 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7feda408d0a0 0x7feda412f2a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:39:07.889 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.883+0000 7f26bf5ae640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f26b806bbc0 0x7f26b808ebe0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:39:07.889 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.883+0000 7f26bf5ae640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f26b8073810 0x7f26b808f120 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:39:07.889 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.883+0000 7f26bf5ae640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f26b8132290 0x7f26b808f660 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:39:07.889 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.884+0000 7f26bf5ae640 1 -- --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7f26b8136960 con 0x7f26b806bbc0 2024-09-14T03:39:07.889 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.884+0000 7f26bf5ae640 1 -- --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7f26b81367e0 con 0x7f26b8073810 2024-09-14T03:39:07.890 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.884+0000 7f26bf5ae640 1 -- --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7f26b8136ae0 con 0x7f26b8132290 2024-09-14T03:39:07.890 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.884+0000 7f26bdb4b640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f26b8132290 0x7f26b808f660 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:39:07.890 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.884+0000 7feda37fe640 1 -- 172.21.15.83:0/3272817285 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7feda408c780 msgr2=0x7feda412ed60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:07.890 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.884+0000 7feda37fe640 1 --2- 172.21.15.83:0/3272817285 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7feda408c780 0x7feda412ed60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:07.890 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.884+0000 7feda37fe640 1 -- 172.21.15.83:0/3272817285 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7feda408d0a0 msgr2=0x7feda412f2a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:07.890 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.884+0000 7feda37fe640 1 --2- 172.21.15.83:0/3272817285 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7feda408d0a0 0x7feda412f2a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:07.890 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.884+0000 7feda37fe640 1 -- 172.21.15.83:0/3272817285 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7feda40924c0 con 0x7feda412e1d0 2024-09-14T03:39:07.891 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.884+0000 7f26bdb4b640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f26b8132290 0x7f26b808f660 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3302/0 says I am v2:172.21.15.83:33072/0 (socket says 172.21.15.83:33072) 2024-09-14T03:39:07.891 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.884+0000 7f26bdb4b640 1 -- 172.21.15.83:0/2376609342 learned_addr learned my addr 172.21.15.83:0/2376609342 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:39:07.891 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.884+0000 7f26bcb49640 1 --2- 172.21.15.83:0/2376609342 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f26b8073810 0x7f26b808f120 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:39:07.891 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.884+0000 7f26bdb4b640 1 -- 172.21.15.83:0/2376609342 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f26b8073810 msgr2=0x7f26b808f120 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:07.891 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.884+0000 7f26bdb4b640 1 --2- 172.21.15.83:0/2376609342 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f26b8073810 0x7f26b808f120 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:07.891 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.884+0000 7f26bdb4b640 1 -- 172.21.15.83:0/2376609342 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f26b806bbc0 msgr2=0x7f26b808ebe0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:07.891 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.884+0000 7f26bdb4b640 1 --2- 172.21.15.83:0/2376609342 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f26b806bbc0 0x7f26b808ebe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:07.892 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.884+0000 7f26bdb4b640 1 -- 172.21.15.83:0/2376609342 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f26b81232f0 con 0x7f26b8132290 2024-09-14T03:39:07.892 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.884+0000 7feda37fe640 1 --2- 172.21.15.83:0/3272817285 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7feda412e1d0 0x7feda412f7e0 secure :-1 s=READY pgs=47 cs=0 l=1 rev1=1 crypto rx=0x7fed9c00c760 tx=0x7fed9c00cc20 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:39:07.892 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.884+0000 7fed8bfff640 1 -- 172.21.15.83:0/3272817285 <== mon.1 v2:172.21.15.83:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fed9c003b70 con 0x7feda412e1d0 2024-09-14T03:39:07.892 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.884+0000 7fed8bfff640 1 -- 172.21.15.83:0/3272817285 <== mon.1 v2:172.21.15.83:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fed9c003d30 con 0x7feda412e1d0 2024-09-14T03:39:07.892 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.884+0000 7fed8bfff640 1 -- 172.21.15.83:0/3272817285 <== mon.1 v2:172.21.15.83:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fed9c00aab0 con 0x7feda412e1d0 2024-09-14T03:39:07.893 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.884+0000 7f26bdb4b640 1 --2- 172.21.15.83:0/2376609342 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f26b8132290 0x7f26b808f660 secure :-1 s=READY pgs=48 cs=0 l=1 rev1=1 crypto rx=0x7f26b000c730 tx=0x7f26b000cbf0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:39:07.893 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.885+0000 7feda99fd640 1 -- 172.21.15.83:0/3272817285 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7feda41312e0 con 0x7feda412e1d0 2024-09-14T03:39:07.893 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.885+0000 7f26adffb640 1 -- 172.21.15.83:0/2376609342 <== mon.2 v2:172.21.15.83:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f26b0003b70 con 0x7f26b8132290 2024-09-14T03:39:07.893 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.885+0000 7f26adffb640 1 -- 172.21.15.83:0/2376609342 <== mon.2 v2:172.21.15.83:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f26b0003d10 con 0x7f26b8132290 2024-09-14T03:39:07.893 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.885+0000 7f26bf5ae640 1 -- 172.21.15.83:0/2376609342 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f26b8122980 con 0x7f26b8132290 2024-09-14T03:39:07.893 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.885+0000 7f26adffb640 1 -- 172.21.15.83:0/2376609342 <== mon.2 v2:172.21.15.83:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f26b000aab0 con 0x7f26b8132290 2024-09-14T03:39:07.894 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.885+0000 7feda99fd640 1 -- 172.21.15.83:0/3272817285 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7feda412ffe0 con 0x7feda412e1d0 2024-09-14T03:39:07.894 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.886+0000 7f26bf5ae640 1 -- 172.21.15.83:0/2376609342 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7f26b8090da0 con 0x7f26b8132290 2024-09-14T03:39:07.894 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.886+0000 7feda99fd640 1 -- 172.21.15.83:0/3272817285 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7feda4130690 con 0x7feda412e1d0 2024-09-14T03:39:07.895 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.886+0000 7f26adffb640 1 -- 172.21.15.83:0/2376609342 <== mon.2 v2:172.21.15.83:3302/0 4 ==== mgrmap(e 5) ==== 55999+0+0 (secure 0 0 0) 0x7f26b000ac50 con 0x7f26b8132290 2024-09-14T03:39:07.895 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.886+0000 7fed8bfff640 1 -- 172.21.15.83:0/3272817285 <== mon.1 v2:172.21.15.83:3301/0 4 ==== mgrmap(e 5) ==== 55999+0+0 (secure 0 0 0) 0x7fed9c00ac50 con 0x7feda412e1d0 2024-09-14T03:39:07.895 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.887+0000 7f26adffb640 1 -- 172.21.15.83:0/2376609342 <== mon.2 v2:172.21.15.83:3302/0 5 ==== osd_map(18..18 src has 1..18) ==== 2868+0+0 (secure 0 0 0) 0x7f26b0062d10 con 0x7f26b8132290 2024-09-14T03:39:07.895 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.887+0000 7f26bf5ae640 1 -- 172.21.15.83:0/2376609342 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f26b806bfa0 con 0x7f26b8132290 2024-09-14T03:39:07.896 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.888+0000 7fed8bfff640 1 -- 172.21.15.83:0/3272817285 <== mon.1 v2:172.21.15.83:3301/0 5 ==== osd_map(18..18 src has 1..18) ==== 2868+0+0 (secure 0 0 0) 0x7fed9c05d700 con 0x7feda412e1d0 2024-09-14T03:39:07.896 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.894+0000 7fed8bfff640 1 -- 172.21.15.83:0/3272817285 <== mon.1 v2:172.21.15.83:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+151260 (secure 0 0 0) 0x7fed9c00ce10 con 0x7feda412e1d0 2024-09-14T03:39:07.896 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.895+0000 7f26adffb640 1 -- 172.21.15.83:0/2376609342 <== mon.2 v2:172.21.15.83:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+151260 (secure 0 0 0) 0x7f26b003c160 con 0x7f26b8132290 2024-09-14T03:39:07.985 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.983+0000 7ff6e892c640 1 -- 172.21.15.83:0/3840682549 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 0} v 0) -- 0x7ff6e4068c60 con 0x7ff6e409b500 2024-09-14T03:39:07.985 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.984+0000 7ff6d2ffd640 1 -- 172.21.15.83:0/3840682549 <== mon.0 v2:172.21.15.83:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 0}]=0 v0) ==== 74+0+12 (secure 0 0 0) 0x7ff6d807d3b0 con 0x7ff6e409b500 2024-09-14T03:39:07.985 INFO:teuthology.orchestra.run.smithi083.stdout:38654705668 2024-09-14T03:39:07.987 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.985+0000 7ff6e892c640 1 -- 172.21.15.83:0/3840682549 >> v1:172.21.15.83:6800/1934165865 conn(0x7ff6c4048000 legacy=0x7ff6c404a4c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:07.987 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.985+0000 7ff6e892c640 1 -- 172.21.15.83:0/3840682549 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7ff6e409b500 msgr2=0x7ff6e4123fd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:07.987 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.985+0000 7ff6e892c640 1 --2- 172.21.15.83:0/3840682549 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7ff6e409b500 0x7ff6e4123fd0 secure :-1 s=READY pgs=82 cs=0 l=1 rev1=1 crypto rx=0x7ff6d8031000 tx=0x7ff6d80038a0 comp rx=0 tx=0).stop 2024-09-14T03:39:07.987 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.985+0000 7ff6e892c640 1 -- 172.21.15.83:0/3840682549 shutdown_connections 2024-09-14T03:39:07.987 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.985+0000 7ff6e892c640 1 --2- 172.21.15.83:0/3840682549 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7ff6e412e720 0x7ff6e41270d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:07.987 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.985+0000 7ff6e892c640 1 --2- 172.21.15.83:0/3840682549 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7ff6e409b500 0x7ff6e4123fd0 unknown :-1 s=CLOSED pgs=82 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:07.987 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.985+0000 7ff6e892c640 1 --2- 172.21.15.83:0/3840682549 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7ff6e4096d40 0x7ff6e4123a90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:07.987 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.985+0000 7ff6e892c640 1 -- 172.21.15.83:0/3840682549 >> 172.21.15.83:0/3840682549 conn(0x7ff6e409e920 msgr2=0x7ff6e41286f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:39:07.987 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.986+0000 7ff6e892c640 1 -- 172.21.15.83:0/3840682549 shutdown_connections 2024-09-14T03:39:07.988 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:07.986+0000 7ff6e892c640 1 -- 172.21.15.83:0/3840682549 wait complete. 2024-09-14T03:39:07.998 INFO:tasks.ceph.ceph_manager.ceph:need seq 38654705668 got 38654705668 for osd.0 2024-09-14T03:39:07.998 DEBUG:teuthology.parallel:result is None 2024-09-14T03:39:08.043 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.042+0000 7f26bf5ae640 1 -- 172.21.15.83:0/2376609342 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 1} v 0) -- 0x7f26b8134df0 con 0x7f26b8132290 2024-09-14T03:39:08.044 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.042+0000 7f26adffb640 1 -- 172.21.15.83:0/2376609342 <== mon.2 v2:172.21.15.83:3302/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 1}]=0 v0) ==== 74+0+12 (secure 0 0 0) 0x7f26b005c040 con 0x7f26b8132290 2024-09-14T03:39:08.044 INFO:teuthology.orchestra.run.smithi083.stdout:47244640260 2024-09-14T03:39:08.045 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.044+0000 7f26bf5ae640 1 -- 172.21.15.83:0/2376609342 >> v1:172.21.15.83:6800/1934165865 conn(0x7f269c047fd0 legacy=0x7f269c04a490 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:08.046 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.044+0000 7f26bf5ae640 1 -- 172.21.15.83:0/2376609342 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f26b8132290 msgr2=0x7f26b808f660 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:08.046 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.044+0000 7f26bf5ae640 1 --2- 172.21.15.83:0/2376609342 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f26b8132290 0x7f26b808f660 secure :-1 s=READY pgs=48 cs=0 l=1 rev1=1 crypto rx=0x7f26b000c730 tx=0x7f26b000cbf0 comp rx=0 tx=0).stop 2024-09-14T03:39:08.046 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.044+0000 7f26bf5ae640 1 -- 172.21.15.83:0/2376609342 shutdown_connections 2024-09-14T03:39:08.046 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.044+0000 7f26bf5ae640 1 --2- 172.21.15.83:0/2376609342 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f26b8132290 0x7f26b808f660 unknown :-1 s=CLOSED pgs=48 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:08.046 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.044+0000 7f26bf5ae640 1 --2- 172.21.15.83:0/2376609342 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f26b8073810 0x7f26b808f120 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:08.046 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.044+0000 7f26bf5ae640 1 --2- 172.21.15.83:0/2376609342 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f26b806bbc0 0x7f26b808ebe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:08.046 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.044+0000 7f26bf5ae640 1 -- 172.21.15.83:0/2376609342 >> 172.21.15.83:0/2376609342 conn(0x7f26b809d510 msgr2=0x7f26b80970b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:39:08.046 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.044+0000 7f26bf5ae640 1 -- 172.21.15.83:0/2376609342 shutdown_connections 2024-09-14T03:39:08.046 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.044+0000 7f26bf5ae640 1 -- 172.21.15.83:0/2376609342 wait complete. 2024-09-14T03:39:08.056 INFO:tasks.ceph.ceph_manager.ceph:need seq 47244640260 got 47244640260 for osd.1 2024-09-14T03:39:08.057 DEBUG:teuthology.parallel:result is None 2024-09-14T03:39:08.064 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.063+0000 7f1cdec67640 1 -- 172.21.15.83:0/1528282311 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 3} v 0) -- 0x7f1cd8098780 con 0x7f1cd812c8f0 2024-09-14T03:39:08.065 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.064+0000 7f1cd57fa640 1 -- 172.21.15.83:0/1528282311 <== mon.1 v2:172.21.15.83:3301/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 3}]=0 v0) ==== 74+0+12 (secure 0 0 0) 0x7f1cd8098780 con 0x7f1cd812c8f0 2024-09-14T03:39:08.065 INFO:teuthology.orchestra.run.smithi083.stdout:47244640260 2024-09-14T03:39:08.067 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.065+0000 7f1cdec67640 1 -- 172.21.15.83:0/1528282311 >> v1:172.21.15.83:6800/1934165865 conn(0x7f1cbc047fd0 legacy=0x7f1cbc04a490 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:08.067 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.065+0000 7f1cdec67640 1 -- 172.21.15.83:0/1528282311 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f1cd812c8f0 msgr2=0x7f1cd80721b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:08.067 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.065+0000 7f1cdec67640 1 --2- 172.21.15.83:0/1528282311 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f1cd812c8f0 0x7f1cd80721b0 secure :-1 s=READY pgs=44 cs=0 l=1 rev1=1 crypto rx=0x7f1cd0034040 tx=0x7f1cd0003e60 comp rx=0 tx=0).stop 2024-09-14T03:39:08.067 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.065+0000 7f1cdec67640 1 -- 172.21.15.83:0/1528282311 shutdown_connections 2024-09-14T03:39:08.067 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.065+0000 7f1cdec67640 1 --2- 172.21.15.83:0/1528282311 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f1cd812c8f0 0x7f1cd80721b0 unknown :-1 s=CLOSED pgs=44 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:08.067 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.065+0000 7f1cdec67640 1 --2- 172.21.15.83:0/1528282311 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f1cd812bfd0 0x7f1cd806eea0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:08.067 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.065+0000 7f1cdec67640 1 --2- 172.21.15.83:0/1528282311 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f1cd809c8f0 0x7f1cd806e960 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:08.067 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.065+0000 7f1cdec67640 1 -- 172.21.15.83:0/1528282311 >> 172.21.15.83:0/1528282311 conn(0x7f1cd809b660 msgr2=0x7f1cd8096670 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:39:08.068 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.065+0000 7f1cdec67640 1 -- 172.21.15.83:0/1528282311 shutdown_connections 2024-09-14T03:39:08.068 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.066+0000 7f1cdec67640 1 -- 172.21.15.83:0/1528282311 wait complete. 2024-09-14T03:39:08.077 INFO:tasks.ceph.ceph_manager.ceph:need seq 47244640260 got 47244640260 for osd.3 2024-09-14T03:39:08.077 DEBUG:teuthology.parallel:result is None 2024-09-14T03:39:08.091 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.090+0000 7feda99fd640 1 -- 172.21.15.83:0/3272817285 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 2} v 0) -- 0x7feda4130f10 con 0x7feda412e1d0 2024-09-14T03:39:08.091 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.090+0000 7fed8bfff640 1 -- 172.21.15.83:0/3272817285 <== mon.1 v2:172.21.15.83:3301/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 2}]=0 v0) ==== 74+0+12 (secure 0 0 0) 0x7fed9c03c970 con 0x7feda412e1d0 2024-09-14T03:39:08.092 INFO:teuthology.orchestra.run.smithi083.stdout:47244640260 2024-09-14T03:39:08.093 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.092+0000 7feda99fd640 1 -- 172.21.15.83:0/3272817285 >> v1:172.21.15.83:6800/1934165865 conn(0x7fed900480a0 legacy=0x7fed9004a560 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:08.094 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.092+0000 7feda99fd640 1 -- 172.21.15.83:0/3272817285 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7feda412e1d0 msgr2=0x7feda412f7e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:08.094 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.092+0000 7feda99fd640 1 --2- 172.21.15.83:0/3272817285 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7feda412e1d0 0x7feda412f7e0 secure :-1 s=READY pgs=47 cs=0 l=1 rev1=1 crypto rx=0x7fed9c00c760 tx=0x7fed9c00cc20 comp rx=0 tx=0).stop 2024-09-14T03:39:08.094 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.092+0000 7feda99fd640 1 -- 172.21.15.83:0/3272817285 shutdown_connections 2024-09-14T03:39:08.094 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.092+0000 7feda99fd640 1 --2- 172.21.15.83:0/3272817285 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7feda412e1d0 0x7feda412f7e0 unknown :-1 s=CLOSED pgs=47 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:08.094 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.092+0000 7feda99fd640 1 --2- 172.21.15.83:0/3272817285 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7feda408d0a0 0x7feda412f2a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:08.094 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.092+0000 7feda99fd640 1 --2- 172.21.15.83:0/3272817285 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7feda408c780 0x7feda412ed60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:08.094 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.092+0000 7feda99fd640 1 -- 172.21.15.83:0/3272817285 >> 172.21.15.83:0/3272817285 conn(0x7feda40a0c80 msgr2=0x7feda406e190 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:39:08.094 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.092+0000 7feda99fd640 1 -- 172.21.15.83:0/3272817285 shutdown_connections 2024-09-14T03:39:08.094 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.092+0000 7feda99fd640 1 -- 172.21.15.83:0/3272817285 wait complete. 2024-09-14T03:39:08.104 INFO:tasks.ceph.ceph_manager.ceph:need seq 47244640260 got 47244640260 for osd.2 2024-09-14T03:39:08.104 DEBUG:teuthology.parallel:result is None 2024-09-14T03:39:08.104 INFO:tasks.ceph.ceph_manager.ceph:waiting for clean 2024-09-14T03:39:08.104 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-09-14T03:39:08.246 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.242+0000 7f44afc67640 1 -- 172.21.15.83:0/979914638 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f44a812b050 msgr2=0x7f44a812b430 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:08.247 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.242+0000 7f44afc67640 1 --2- 172.21.15.83:0/979914638 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f44a812b050 0x7f44a812b430 secure :-1 s=READY pgs=48 cs=0 l=1 rev1=1 crypto rx=0x7f449c009890 tx=0x7f449c02a410 comp rx=0 tx=0).stop 2024-09-14T03:39:08.247 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.243+0000 7f44afc67640 1 -- 172.21.15.83:0/979914638 shutdown_connections 2024-09-14T03:39:08.247 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.243+0000 7f44afc67640 1 --2- 172.21.15.83:0/979914638 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f44a8094e80 0x7f44a80952c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:08.247 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.243+0000 7f44afc67640 1 --2- 172.21.15.83:0/979914638 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f44a812b050 0x7f44a812b430 unknown :-1 s=CLOSED pgs=48 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:08.247 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.243+0000 7f44afc67640 1 -- 172.21.15.83:0/979914638 >> 172.21.15.83:0/979914638 conn(0x7f44a8099450 msgr2=0x7f44a80969e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:39:08.247 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.243+0000 7f44afc67640 1 -- 172.21.15.83:0/979914638 shutdown_connections 2024-09-14T03:39:08.247 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.243+0000 7f44afc67640 1 -- 172.21.15.83:0/979914638 wait complete. 2024-09-14T03:39:08.247 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.244+0000 7f44afc67640 1 Processor -- start 2024-09-14T03:39:08.247 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.244+0000 7f44afc67640 1 -- start start 2024-09-14T03:39:08.247 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.244+0000 7f44afc67640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f44a8094e80 0x7f44a81c20c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:39:08.247 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.244+0000 7f44afc67640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f44a8095800 0x7f44a81c2600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:39:08.247 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.244+0000 7f44afc67640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f44a812b050 0x7f44a80714c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:39:08.248 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.244+0000 7f44afc67640 1 -- --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7f44a8133970 con 0x7f44a8094e80 2024-09-14T03:39:08.248 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.244+0000 7f44afc67640 1 -- --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7f44a81337f0 con 0x7f44a8095800 2024-09-14T03:39:08.248 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.244+0000 7f44afc67640 1 -- --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7f44a8133af0 con 0x7f44a812b050 2024-09-14T03:39:08.248 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.244+0000 7f44ad202640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f44a8095800 0x7f44a81c2600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:39:08.248 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.244+0000 7f44ada03640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f44a8094e80 0x7f44a81c20c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:39:08.248 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.244+0000 7f44ae204640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f44a812b050 0x7f44a80714c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:39:08.248 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.244+0000 7f44ad202640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f44a8095800 0x7f44a81c2600 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3301/0 says I am v2:172.21.15.83:60778/0 (socket says 172.21.15.83:60778) 2024-09-14T03:39:08.248 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.244+0000 7f44ada03640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f44a8094e80 0x7f44a81c20c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3300/0 says I am v2:172.21.15.83:36080/0 (socket says 172.21.15.83:36080) 2024-09-14T03:39:08.248 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.244+0000 7f44ad202640 1 -- 172.21.15.83:0/1119851280 learned_addr learned my addr 172.21.15.83:0/1119851280 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:39:08.248 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.244+0000 7f44ae204640 1 -- 172.21.15.83:0/1119851280 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f44a8095800 msgr2=0x7f44a81c2600 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:08.248 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.244+0000 7f44ae204640 1 --2- 172.21.15.83:0/1119851280 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f44a8095800 0x7f44a81c2600 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:08.248 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.244+0000 7f44ae204640 1 -- 172.21.15.83:0/1119851280 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f44a8094e80 msgr2=0x7f44a81c20c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:08.248 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.244+0000 7f44ae204640 1 --2- 172.21.15.83:0/1119851280 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f44a8094e80 0x7f44a81c20c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:08.249 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.244+0000 7f44ae204640 1 -- 172.21.15.83:0/1119851280 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f44a812bce0 con 0x7f44a812b050 2024-09-14T03:39:08.249 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.245+0000 7f44ae204640 1 --2- 172.21.15.83:0/1119851280 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f44a812b050 0x7f44a80714c0 secure :-1 s=READY pgs=49 cs=0 l=1 rev1=1 crypto rx=0x7f44a000cb40 tx=0x7f44a000a060 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:39:08.249 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.245+0000 7f449a7fc640 1 -- 172.21.15.83:0/1119851280 <== mon.2 v2:172.21.15.83:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f44a0016020 con 0x7f44a812b050 2024-09-14T03:39:08.249 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.245+0000 7f449a7fc640 1 -- 172.21.15.83:0/1119851280 <== mon.2 v2:172.21.15.83:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f44a000ad30 con 0x7f44a812b050 2024-09-14T03:39:08.249 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.245+0000 7f44afc67640 1 -- 172.21.15.83:0/1119851280 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f44a81c2b40 con 0x7f44a812b050 2024-09-14T03:39:08.249 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.245+0000 7f449a7fc640 1 -- 172.21.15.83:0/1119851280 <== mon.2 v2:172.21.15.83:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f44a0012680 con 0x7f44a812b050 2024-09-14T03:39:08.249 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.245+0000 7f44afc67640 1 -- 172.21.15.83:0/1119851280 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7f44a812c750 con 0x7f44a812b050 2024-09-14T03:39:08.250 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.246+0000 7f449a7fc640 1 -- 172.21.15.83:0/1119851280 <== mon.2 v2:172.21.15.83:3302/0 4 ==== mgrmap(e 5) ==== 55999+0+0 (secure 0 0 0) 0x7f44a0019020 con 0x7f44a812b050 2024-09-14T03:39:08.250 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.246+0000 7f44afc67640 1 -- 172.21.15.83:0/1119851280 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f44a812d210 con 0x7f44a812b050 2024-09-14T03:39:08.250 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.246+0000 7f449a7fc640 1 -- 172.21.15.83:0/1119851280 <== mon.2 v2:172.21.15.83:3302/0 5 ==== osd_map(18..18 src has 1..18) ==== 2868+0+0 (secure 0 0 0) 0x7f44a005dc90 con 0x7f44a812b050 2024-09-14T03:39:08.252 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.250+0000 7f449a7fc640 1 -- 172.21.15.83:0/1119851280 <== mon.2 v2:172.21.15.83:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+151260 (secure 0 0 0) 0x7f44a812c750 con 0x7f44a812b050 2024-09-14T03:39:08.388 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.386+0000 7f44afc67640 1 -- 172.21.15.83:0/1119851280 --> v1:172.21.15.83:6800/1934165865 -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) -- 0x7f44a812b640 con 0x7f448c047fd0 2024-09-14T03:39:08.389 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.387+0000 7f449a7fc640 1 -- 172.21.15.83:0/1119851280 <== mgr.4146 v1:172.21.15.83:6800/1934165865 1 ==== mgr_command_reply(tid 0: 0 dumped all) ==== 18+0+32233 (unknown 2965378022 0 3886086923) 0x7f44a812b640 con 0x7f448c047fd0 2024-09-14T03:39:08.389 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:39:08.389 INFO:teuthology.orchestra.run.smithi083.stderr:dumped all 2024-09-14T03:39:08.390 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.389+0000 7f44afc67640 1 -- 172.21.15.83:0/1119851280 >> v1:172.21.15.83:6800/1934165865 conn(0x7f448c047fd0 legacy=0x7f448c04a490 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:08.391 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.389+0000 7f44afc67640 1 -- 172.21.15.83:0/1119851280 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f44a812b050 msgr2=0x7f44a80714c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:08.391 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.389+0000 7f44afc67640 1 --2- 172.21.15.83:0/1119851280 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f44a812b050 0x7f44a80714c0 secure :-1 s=READY pgs=49 cs=0 l=1 rev1=1 crypto rx=0x7f44a000cb40 tx=0x7f44a000a060 comp rx=0 tx=0).stop 2024-09-14T03:39:08.391 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.389+0000 7f44afc67640 1 -- 172.21.15.83:0/1119851280 shutdown_connections 2024-09-14T03:39:08.391 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.389+0000 7f44afc67640 1 --2- 172.21.15.83:0/1119851280 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f44a812b050 0x7f44a80714c0 unknown :-1 s=CLOSED pgs=49 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:08.391 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.389+0000 7f44afc67640 1 --2- 172.21.15.83:0/1119851280 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f44a8095800 0x7f44a81c2600 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:08.391 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.389+0000 7f44afc67640 1 --2- 172.21.15.83:0/1119851280 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f44a8094e80 0x7f44a81c20c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:08.391 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.389+0000 7f44afc67640 1 -- 172.21.15.83:0/1119851280 >> 172.21.15.83:0/1119851280 conn(0x7f44a8099450 msgr2=0x7f44a8094590 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:39:08.391 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.389+0000 7f44afc67640 1 -- 172.21.15.83:0/1119851280 shutdown_connections 2024-09-14T03:39:08.391 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.389+0000 7f44afc67640 1 -- 172.21.15.83:0/1119851280 wait complete. 2024-09-14T03:39:08.401 INFO:teuthology.orchestra.run.smithi083.stdout:{"pg_ready":true,"pg_map":{"version":22,"stamp":"2024-09-14T03:39:06.893263+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":139,"num_write_kb":1896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":100,"ondisk_log_size":100,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":10,"num_osds":4,"num_per_pool_osds":4,"num_per_pool_omap_osds":4,"kb":377487360,"kb_used":109900,"kb_used_data":1776,"kb_used_omap":6,"kb_used_meta":108025,"kb_avail":377377460,"statfs":{"total":386547056640,"available":386434519040,"internally_reserved":0,"allocated":1818624,"data_stored":1353470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6378,"internal_metadata":110618390},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"4.086805"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":19,"reported_epoch":18,"state":"active+clean","last_fresh":"2024-09-14T03:39:03.811399+0000","last_change":"2024-09-14T03:39:03.811650+0000","last_active":"2024-09-14T03:39:03.811399+0000","last_peered":"2024-09-14T03:39:03.811399+0000","last_clean":"2024-09-14T03:39:03.811399+0000","last_became_active":"2024-09-14T03:39:01.978698+0000","last_became_peered":"2024-09-14T03:39:01.978698+0000","last_unstale":"2024-09-14T03:39:03.811399+0000","last_undegraded":"2024-09-14T03:39:03.811399+0000","last_fullsized":"2024-09-14T03:39:03.811399+0000","mapping_epoch":15,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":16,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_clean_scrub_stamp":"2024-09-14T03:39:00.791161+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:14:21.446815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00098140899999999993,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.6","version":"0'0","reported_seq":19,"reported_epoch":18,"state":"active+clean","last_fresh":"2024-09-14T03:39:03.819578+0000","last_change":"2024-09-14T03:39:03.819819+0000","last_active":"2024-09-14T03:39:03.819578+0000","last_peered":"2024-09-14T03:39:03.819578+0000","last_clean":"2024-09-14T03:39:03.819578+0000","last_became_active":"2024-09-14T03:39:01.978827+0000","last_became_peered":"2024-09-14T03:39:01.978827+0000","last_unstale":"2024-09-14T03:39:03.819578+0000","last_undegraded":"2024-09-14T03:39:03.819578+0000","last_fullsized":"2024-09-14T03:39:03.819578+0000","mapping_epoch":15,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":16,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_clean_scrub_stamp":"2024-09-14T03:39:00.791161+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:40:25.557607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00045411099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.5","version":"0'0","reported_seq":19,"reported_epoch":18,"state":"active+clean","last_fresh":"2024-09-14T03:39:03.811449+0000","last_change":"2024-09-14T03:39:03.811665+0000","last_active":"2024-09-14T03:39:03.811449+0000","last_peered":"2024-09-14T03:39:03.811449+0000","last_clean":"2024-09-14T03:39:03.811449+0000","last_became_active":"2024-09-14T03:39:01.978728+0000","last_became_peered":"2024-09-14T03:39:01.978728+0000","last_unstale":"2024-09-14T03:39:03.811449+0000","last_undegraded":"2024-09-14T03:39:03.811449+0000","last_fullsized":"2024-09-14T03:39:03.811449+0000","mapping_epoch":15,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":16,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_clean_scrub_stamp":"2024-09-14T03:39:00.791161+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:34:31.446372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00087291699999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.4","version":"0'0","reported_seq":19,"reported_epoch":18,"state":"active+clean","last_fresh":"2024-09-14T03:39:03.819635+0000","last_change":"2024-09-14T03:39:03.819854+0000","last_active":"2024-09-14T03:39:03.819635+0000","last_peered":"2024-09-14T03:39:03.819635+0000","last_clean":"2024-09-14T03:39:03.819635+0000","last_became_active":"2024-09-14T03:39:01.978754+0000","last_became_peered":"2024-09-14T03:39:01.978754+0000","last_unstale":"2024-09-14T03:39:03.819635+0000","last_undegraded":"2024-09-14T03:39:03.819635+0000","last_fullsized":"2024-09-14T03:39:03.819635+0000","mapping_epoch":15,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":16,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_clean_scrub_stamp":"2024-09-14T03:39:00.791161+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:12:43.787489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00048052199999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.2","version":"18'2","reported_seq":21,"reported_epoch":18,"state":"active+clean","last_fresh":"2024-09-14T03:39:03.812269+0000","last_change":"2024-09-14T03:39:03.812269+0000","last_active":"2024-09-14T03:39:03.812269+0000","last_peered":"2024-09-14T03:39:03.812269+0000","last_clean":"2024-09-14T03:39:03.812269+0000","last_became_active":"2024-09-14T03:39:01.808874+0000","last_became_peered":"2024-09-14T03:39:01.808874+0000","last_unstale":"2024-09-14T03:39:03.812269+0000","last_undegraded":"2024-09-14T03:39:03.812269+0000","last_fullsized":"2024-09-14T03:39:03.812269+0000","mapping_epoch":15,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":16,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_clean_scrub_stamp":"2024-09-14T03:39:00.791161+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:44:11.882619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00135844,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.1","version":"0'0","reported_seq":19,"reported_epoch":18,"state":"active+clean","last_fresh":"2024-09-14T03:39:03.811368+0000","last_change":"2024-09-14T03:39:03.811664+0000","last_active":"2024-09-14T03:39:03.811368+0000","last_peered":"2024-09-14T03:39:03.811368+0000","last_clean":"2024-09-14T03:39:03.811368+0000","last_became_active":"2024-09-14T03:39:01.807757+0000","last_became_peered":"2024-09-14T03:39:01.807757+0000","last_unstale":"2024-09-14T03:39:03.811368+0000","last_undegraded":"2024-09-14T03:39:03.811368+0000","last_fullsized":"2024-09-14T03:39:03.811368+0000","mapping_epoch":15,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":16,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_clean_scrub_stamp":"2024-09-14T03:39:00.791161+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:55:28.670915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046880499999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.0","version":"0'0","reported_seq":19,"reported_epoch":18,"state":"active+clean","last_fresh":"2024-09-14T03:39:03.810997+0000","last_change":"2024-09-14T03:39:03.811316+0000","last_active":"2024-09-14T03:39:03.810997+0000","last_peered":"2024-09-14T03:39:03.810997+0000","last_clean":"2024-09-14T03:39:03.810997+0000","last_became_active":"2024-09-14T03:39:01.810839+0000","last_became_peered":"2024-09-14T03:39:01.810839+0000","last_unstale":"2024-09-14T03:39:03.810997+0000","last_undegraded":"2024-09-14T03:39:03.810997+0000","last_fullsized":"2024-09-14T03:39:03.810997+0000","mapping_epoch":15,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":16,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_clean_scrub_stamp":"2024-09-14T03:39:00.791161+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:53:45.021355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000765802,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.3","version":"16'1","reported_seq":20,"reported_epoch":18,"state":"active+clean","last_fresh":"2024-09-14T03:39:03.811072+0000","last_change":"2024-09-14T03:39:03.811340+0000","last_active":"2024-09-14T03:39:03.811072+0000","last_peered":"2024-09-14T03:39:03.811072+0000","last_clean":"2024-09-14T03:39:03.811072+0000","last_became_active":"2024-09-14T03:39:01.809115+0000","last_became_peered":"2024-09-14T03:39:01.809115+0000","last_unstale":"2024-09-14T03:39:03.811072+0000","last_undegraded":"2024-09-14T03:39:03.811072+0000","last_fullsized":"2024-09-14T03:39:03.811072+0000","mapping_epoch":15,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":16,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_clean_scrub_stamp":"2024-09-14T03:39:00.791161+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:17:21.729224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00075472199999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"1.0","version":"13'97","reported_seq":147,"reported_epoch":18,"state":"active+clean","last_fresh":"2024-09-14T03:39:03.819344+0000","last_change":"2024-09-14T03:38:59.008461+0000","last_active":"2024-09-14T03:39:03.819344+0000","last_peered":"2024-09-14T03:39:03.819344+0000","last_clean":"2024-09-14T03:39:03.819344+0000","last_became_active":"2024-09-14T03:38:59.006003+0000","last_became_peered":"2024-09-14T03:38:59.006003+0000","last_unstale":"2024-09-14T03:39:03.819344+0000","last_undegraded":"2024-09-14T03:39:03.819344+0000","last_fullsized":"2024-09-14T03:39:03.819344+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:38:57.773685+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:57.773685+0000","last_clean_scrub_stamp":"2024-09-14T03:38:57.773685+0000","objects_scrubbed":0,"log_size":97,"log_dups_size":0,"ondisk_log_size":97,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:21:23.620251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":137,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":4},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":137,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":97,"ondisk_log_size":97,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":3,"up_from":11,"seq":47244640260,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27184,"kb_used_data":156,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344656,"statfs":{"total":96636764160,"available":96608927744,"internally_reserved":0,"allocated":159744,"data_stored":43193,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":27654599},"hb_peers":[0,1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":11,"seq":47244640260,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27188,"kb_used_data":152,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344652,"statfs":{"total":96636764160,"available":96608923648,"internally_reserved":0,"allocated":155648,"data_stored":43174,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,1,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":11,"seq":47244640260,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27768,"kb_used_data":736,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344072,"statfs":{"total":96636764160,"available":96608329728,"internally_reserved":0,"allocated":753664,"data_stored":633561,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,2,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":0,"up_from":9,"seq":38654705668,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27760,"kb_used_data":732,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344080,"statfs":{"total":96636764160,"available":96608337920,"internally_reserved":0,"allocated":749568,"data_stored":633542,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[1,2,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T03:39:08.402 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-09-14T03:39:08.502 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.500+0000 7f2f36133640 1 -- 172.21.15.83:0/2866295796 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f2f30095f50 msgr2=0x7f2f30096330 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:08.502 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.500+0000 7f2f36133640 1 --2- 172.21.15.83:0/2866295796 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f2f30095f50 0x7f2f30096330 secure :-1 s=READY pgs=50 cs=0 l=1 rev1=1 crypto rx=0x7f2f20009860 tx=0x7f2f2002a3d0 comp rx=0 tx=0).stop 2024-09-14T03:39:08.502 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.500+0000 7f2f36133640 1 -- 172.21.15.83:0/2866295796 shutdown_connections 2024-09-14T03:39:08.502 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.500+0000 7f2f36133640 1 --2- 172.21.15.83:0/2866295796 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f2f3012c1d0 0x7f2f3012c620 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:08.502 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.500+0000 7f2f36133640 1 --2- 172.21.15.83:0/2866295796 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f2f30095f50 0x7f2f30096330 unknown :-1 s=CLOSED pgs=50 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:08.502 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.500+0000 7f2f36133640 1 -- 172.21.15.83:0/2866295796 >> 172.21.15.83:0/2866295796 conn(0x7f2f3009c590 msgr2=0x7f2f3009c9a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:39:08.502 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.500+0000 7f2f36133640 1 -- 172.21.15.83:0/2866295796 shutdown_connections 2024-09-14T03:39:08.503 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.500+0000 7f2f36133640 1 -- 172.21.15.83:0/2866295796 wait complete. 2024-09-14T03:39:08.505 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.501+0000 7f2f36133640 1 Processor -- start 2024-09-14T03:39:08.505 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.501+0000 7f2f36133640 1 -- start start 2024-09-14T03:39:08.505 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.501+0000 7f2f36133640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f2f30095f50 0x7f2f3006f9d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:39:08.505 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.501+0000 7f2f36133640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f2f30096870 0x7f2f3006ff10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:39:08.505 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.501+0000 7f2f36133640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f2f3012c1d0 0x7f2f30070450 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:39:08.505 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.502+0000 7f2f36133640 1 -- --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7f2f30134790 con 0x7f2f3012c1d0 2024-09-14T03:39:08.505 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.502+0000 7f2f36133640 1 -- --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7f2f30134610 con 0x7f2f30095f50 2024-09-14T03:39:08.505 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.502+0000 7f2f36133640 1 -- --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7f2f30134910 con 0x7f2f30096870 2024-09-14T03:39:08.505 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.502+0000 7f2f2f7fe640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f2f30095f50 0x7f2f3006f9d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:39:08.505 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.502+0000 7f2f2effd640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f2f30096870 0x7f2f3006ff10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:39:08.505 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.502+0000 7f2f2effd640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f2f30096870 0x7f2f3006ff10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3302/0 says I am v2:172.21.15.83:33102/0 (socket says 172.21.15.83:33102) 2024-09-14T03:39:08.505 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.502+0000 7f2f2f7fe640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f2f30095f50 0x7f2f3006f9d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3301/0 says I am v2:172.21.15.83:60788/0 (socket says 172.21.15.83:60788) 2024-09-14T03:39:08.505 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.502+0000 7f2f2effd640 1 -- 172.21.15.83:0/2525512220 learned_addr learned my addr 172.21.15.83:0/2525512220 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:39:08.506 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.502+0000 7f2f2ffff640 1 --2- 172.21.15.83:0/2525512220 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f2f3012c1d0 0x7f2f30070450 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:39:08.506 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.502+0000 7f2f2effd640 1 -- 172.21.15.83:0/2525512220 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f2f30095f50 msgr2=0x7f2f3006f9d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:08.506 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.502+0000 7f2f2effd640 1 --2- 172.21.15.83:0/2525512220 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f2f30095f50 0x7f2f3006f9d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:08.506 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.502+0000 7f2f2effd640 1 -- 172.21.15.83:0/2525512220 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f2f3012c1d0 msgr2=0x7f2f30070450 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:08.506 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.502+0000 7f2f2effd640 1 --2- 172.21.15.83:0/2525512220 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f2f3012c1d0 0x7f2f30070450 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:08.506 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.502+0000 7f2f2effd640 1 -- 172.21.15.83:0/2525512220 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f2f30070f50 con 0x7f2f30096870 2024-09-14T03:39:08.506 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.502+0000 7f2f2f7fe640 1 --2- 172.21.15.83:0/2525512220 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f2f30095f50 0x7f2f3006f9d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:39:08.506 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.502+0000 7f2f2ffff640 1 --2- 172.21.15.83:0/2525512220 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f2f3012c1d0 0x7f2f30070450 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:39:08.506 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.502+0000 7f2f2effd640 1 --2- 172.21.15.83:0/2525512220 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f2f30096870 0x7f2f3006ff10 secure :-1 s=READY pgs=51 cs=0 l=1 rev1=1 crypto rx=0x7f2f2400c490 tx=0x7f2f2400c890 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:39:08.506 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.502+0000 7f2f17fff640 1 -- 172.21.15.83:0/2525512220 <== mon.2 v2:172.21.15.83:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2f24011790 con 0x7f2f30096870 2024-09-14T03:39:08.506 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.502+0000 7f2f17fff640 1 -- 172.21.15.83:0/2525512220 <== mon.2 v2:172.21.15.83:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f2f24017070 con 0x7f2f30096870 2024-09-14T03:39:08.506 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.502+0000 7f2f36133640 1 -- 172.21.15.83:0/2525512220 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f2f30071830 con 0x7f2f30096870 2024-09-14T03:39:08.506 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.502+0000 7f2f17fff640 1 -- 172.21.15.83:0/2525512220 <== mon.2 v2:172.21.15.83:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2f24011d00 con 0x7f2f30096870 2024-09-14T03:39:08.506 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.502+0000 7f2f36133640 1 -- 172.21.15.83:0/2525512220 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7f2f30072030 con 0x7f2f30096870 2024-09-14T03:39:08.506 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.503+0000 7f2f36133640 1 -- 172.21.15.83:0/2525512220 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f2f30072260 con 0x7f2f30096870 2024-09-14T03:39:08.507 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.503+0000 7f2f17fff640 1 -- 172.21.15.83:0/2525512220 <== mon.2 v2:172.21.15.83:3302/0 4 ==== mgrmap(e 5) ==== 55999+0+0 (secure 0 0 0) 0x7f2f2401c460 con 0x7f2f30096870 2024-09-14T03:39:08.507 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.504+0000 7f2f17fff640 1 -- 172.21.15.83:0/2525512220 <== mon.2 v2:172.21.15.83:3302/0 5 ==== osd_map(18..18 src has 1..18) ==== 2868+0+0 (secure 0 0 0) 0x7f2f2405d760 con 0x7f2f30096870 2024-09-14T03:39:08.508 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.507+0000 7f2f17fff640 1 -- 172.21.15.83:0/2525512220 <== mon.2 v2:172.21.15.83:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+151260 (secure 0 0 0) 0x7f2f2405c8b0 con 0x7f2f30096870 2024-09-14T03:39:08.647 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.646+0000 7f2f36133640 1 -- 172.21.15.83:0/2525512220 --> v1:172.21.15.83:6800/1934165865 -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) -- 0x7f2f30096330 con 0x7f2f1c047fd0 2024-09-14T03:39:08.649 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.647+0000 7f2f17fff640 1 -- 172.21.15.83:0/2525512220 <== mgr.4146 v1:172.21.15.83:6800/1934165865 1 ==== mgr_command_reply(tid 0: 0 dumped all) ==== 18+0+32233 (unknown 2965378022 0 3886086923) 0x7f2f30096330 con 0x7f2f1c047fd0 2024-09-14T03:39:08.650 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:39:08.651 INFO:teuthology.orchestra.run.smithi083.stderr:dumped all 2024-09-14T03:39:08.651 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.649+0000 7f2f36133640 1 -- 172.21.15.83:0/2525512220 >> v1:172.21.15.83:6800/1934165865 conn(0x7f2f1c047fd0 legacy=0x7f2f1c04a490 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:08.651 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.649+0000 7f2f36133640 1 -- 172.21.15.83:0/2525512220 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f2f30096870 msgr2=0x7f2f3006ff10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:08.651 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.649+0000 7f2f36133640 1 --2- 172.21.15.83:0/2525512220 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f2f30096870 0x7f2f3006ff10 secure :-1 s=READY pgs=51 cs=0 l=1 rev1=1 crypto rx=0x7f2f2400c490 tx=0x7f2f2400c890 comp rx=0 tx=0).stop 2024-09-14T03:39:08.651 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.649+0000 7f2f36133640 1 -- 172.21.15.83:0/2525512220 shutdown_connections 2024-09-14T03:39:08.651 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.649+0000 7f2f36133640 1 --2- 172.21.15.83:0/2525512220 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f2f3012c1d0 0x7f2f30070450 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:08.651 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.649+0000 7f2f36133640 1 --2- 172.21.15.83:0/2525512220 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f2f30096870 0x7f2f3006ff10 unknown :-1 s=CLOSED pgs=51 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:08.651 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.649+0000 7f2f36133640 1 --2- 172.21.15.83:0/2525512220 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f2f30095f50 0x7f2f3006f9d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:08.652 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.649+0000 7f2f36133640 1 -- 172.21.15.83:0/2525512220 >> 172.21.15.83:0/2525512220 conn(0x7f2f3009c590 msgr2=0x7f2f30129090 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:39:08.652 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.649+0000 7f2f36133640 1 -- 172.21.15.83:0/2525512220 shutdown_connections 2024-09-14T03:39:08.652 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.649+0000 7f2f36133640 1 -- 172.21.15.83:0/2525512220 wait complete. 2024-09-14T03:39:08.661 INFO:teuthology.orchestra.run.smithi083.stdout:{"pg_ready":true,"pg_map":{"version":22,"stamp":"2024-09-14T03:39:06.893263+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":139,"num_write_kb":1896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":100,"ondisk_log_size":100,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":10,"num_osds":4,"num_per_pool_osds":4,"num_per_pool_omap_osds":4,"kb":377487360,"kb_used":109900,"kb_used_data":1776,"kb_used_omap":6,"kb_used_meta":108025,"kb_avail":377377460,"statfs":{"total":386547056640,"available":386434519040,"internally_reserved":0,"allocated":1818624,"data_stored":1353470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6378,"internal_metadata":110618390},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"4.086805"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":19,"reported_epoch":18,"state":"active+clean","last_fresh":"2024-09-14T03:39:03.811399+0000","last_change":"2024-09-14T03:39:03.811650+0000","last_active":"2024-09-14T03:39:03.811399+0000","last_peered":"2024-09-14T03:39:03.811399+0000","last_clean":"2024-09-14T03:39:03.811399+0000","last_became_active":"2024-09-14T03:39:01.978698+0000","last_became_peered":"2024-09-14T03:39:01.978698+0000","last_unstale":"2024-09-14T03:39:03.811399+0000","last_undegraded":"2024-09-14T03:39:03.811399+0000","last_fullsized":"2024-09-14T03:39:03.811399+0000","mapping_epoch":15,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":16,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_clean_scrub_stamp":"2024-09-14T03:39:00.791161+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:14:21.446815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00098140899999999993,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.6","version":"0'0","reported_seq":19,"reported_epoch":18,"state":"active+clean","last_fresh":"2024-09-14T03:39:03.819578+0000","last_change":"2024-09-14T03:39:03.819819+0000","last_active":"2024-09-14T03:39:03.819578+0000","last_peered":"2024-09-14T03:39:03.819578+0000","last_clean":"2024-09-14T03:39:03.819578+0000","last_became_active":"2024-09-14T03:39:01.978827+0000","last_became_peered":"2024-09-14T03:39:01.978827+0000","last_unstale":"2024-09-14T03:39:03.819578+0000","last_undegraded":"2024-09-14T03:39:03.819578+0000","last_fullsized":"2024-09-14T03:39:03.819578+0000","mapping_epoch":15,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":16,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_clean_scrub_stamp":"2024-09-14T03:39:00.791161+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:40:25.557607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00045411099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.5","version":"0'0","reported_seq":19,"reported_epoch":18,"state":"active+clean","last_fresh":"2024-09-14T03:39:03.811449+0000","last_change":"2024-09-14T03:39:03.811665+0000","last_active":"2024-09-14T03:39:03.811449+0000","last_peered":"2024-09-14T03:39:03.811449+0000","last_clean":"2024-09-14T03:39:03.811449+0000","last_became_active":"2024-09-14T03:39:01.978728+0000","last_became_peered":"2024-09-14T03:39:01.978728+0000","last_unstale":"2024-09-14T03:39:03.811449+0000","last_undegraded":"2024-09-14T03:39:03.811449+0000","last_fullsized":"2024-09-14T03:39:03.811449+0000","mapping_epoch":15,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":16,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_clean_scrub_stamp":"2024-09-14T03:39:00.791161+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:34:31.446372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00087291699999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.4","version":"0'0","reported_seq":19,"reported_epoch":18,"state":"active+clean","last_fresh":"2024-09-14T03:39:03.819635+0000","last_change":"2024-09-14T03:39:03.819854+0000","last_active":"2024-09-14T03:39:03.819635+0000","last_peered":"2024-09-14T03:39:03.819635+0000","last_clean":"2024-09-14T03:39:03.819635+0000","last_became_active":"2024-09-14T03:39:01.978754+0000","last_became_peered":"2024-09-14T03:39:01.978754+0000","last_unstale":"2024-09-14T03:39:03.819635+0000","last_undegraded":"2024-09-14T03:39:03.819635+0000","last_fullsized":"2024-09-14T03:39:03.819635+0000","mapping_epoch":15,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":16,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_clean_scrub_stamp":"2024-09-14T03:39:00.791161+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:12:43.787489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00048052199999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.2","version":"18'2","reported_seq":21,"reported_epoch":18,"state":"active+clean","last_fresh":"2024-09-14T03:39:03.812269+0000","last_change":"2024-09-14T03:39:03.812269+0000","last_active":"2024-09-14T03:39:03.812269+0000","last_peered":"2024-09-14T03:39:03.812269+0000","last_clean":"2024-09-14T03:39:03.812269+0000","last_became_active":"2024-09-14T03:39:01.808874+0000","last_became_peered":"2024-09-14T03:39:01.808874+0000","last_unstale":"2024-09-14T03:39:03.812269+0000","last_undegraded":"2024-09-14T03:39:03.812269+0000","last_fullsized":"2024-09-14T03:39:03.812269+0000","mapping_epoch":15,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":16,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_clean_scrub_stamp":"2024-09-14T03:39:00.791161+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:44:11.882619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00135844,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.1","version":"0'0","reported_seq":19,"reported_epoch":18,"state":"active+clean","last_fresh":"2024-09-14T03:39:03.811368+0000","last_change":"2024-09-14T03:39:03.811664+0000","last_active":"2024-09-14T03:39:03.811368+0000","last_peered":"2024-09-14T03:39:03.811368+0000","last_clean":"2024-09-14T03:39:03.811368+0000","last_became_active":"2024-09-14T03:39:01.807757+0000","last_became_peered":"2024-09-14T03:39:01.807757+0000","last_unstale":"2024-09-14T03:39:03.811368+0000","last_undegraded":"2024-09-14T03:39:03.811368+0000","last_fullsized":"2024-09-14T03:39:03.811368+0000","mapping_epoch":15,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":16,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_clean_scrub_stamp":"2024-09-14T03:39:00.791161+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:55:28.670915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046880499999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.0","version":"0'0","reported_seq":19,"reported_epoch":18,"state":"active+clean","last_fresh":"2024-09-14T03:39:03.810997+0000","last_change":"2024-09-14T03:39:03.811316+0000","last_active":"2024-09-14T03:39:03.810997+0000","last_peered":"2024-09-14T03:39:03.810997+0000","last_clean":"2024-09-14T03:39:03.810997+0000","last_became_active":"2024-09-14T03:39:01.810839+0000","last_became_peered":"2024-09-14T03:39:01.810839+0000","last_unstale":"2024-09-14T03:39:03.810997+0000","last_undegraded":"2024-09-14T03:39:03.810997+0000","last_fullsized":"2024-09-14T03:39:03.810997+0000","mapping_epoch":15,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":16,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_clean_scrub_stamp":"2024-09-14T03:39:00.791161+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:53:45.021355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000765802,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.3","version":"16'1","reported_seq":20,"reported_epoch":18,"state":"active+clean","last_fresh":"2024-09-14T03:39:03.811072+0000","last_change":"2024-09-14T03:39:03.811340+0000","last_active":"2024-09-14T03:39:03.811072+0000","last_peered":"2024-09-14T03:39:03.811072+0000","last_clean":"2024-09-14T03:39:03.811072+0000","last_became_active":"2024-09-14T03:39:01.809115+0000","last_became_peered":"2024-09-14T03:39:01.809115+0000","last_unstale":"2024-09-14T03:39:03.811072+0000","last_undegraded":"2024-09-14T03:39:03.811072+0000","last_fullsized":"2024-09-14T03:39:03.811072+0000","mapping_epoch":15,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":16,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_clean_scrub_stamp":"2024-09-14T03:39:00.791161+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:17:21.729224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00075472199999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"1.0","version":"13'97","reported_seq":147,"reported_epoch":18,"state":"active+clean","last_fresh":"2024-09-14T03:39:03.819344+0000","last_change":"2024-09-14T03:38:59.008461+0000","last_active":"2024-09-14T03:39:03.819344+0000","last_peered":"2024-09-14T03:39:03.819344+0000","last_clean":"2024-09-14T03:39:03.819344+0000","last_became_active":"2024-09-14T03:38:59.006003+0000","last_became_peered":"2024-09-14T03:38:59.006003+0000","last_unstale":"2024-09-14T03:39:03.819344+0000","last_undegraded":"2024-09-14T03:39:03.819344+0000","last_fullsized":"2024-09-14T03:39:03.819344+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:38:57.773685+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:57.773685+0000","last_clean_scrub_stamp":"2024-09-14T03:38:57.773685+0000","objects_scrubbed":0,"log_size":97,"log_dups_size":0,"ondisk_log_size":97,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:21:23.620251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":137,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":4},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":137,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":97,"ondisk_log_size":97,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":3,"up_from":11,"seq":47244640260,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27184,"kb_used_data":156,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344656,"statfs":{"total":96636764160,"available":96608927744,"internally_reserved":0,"allocated":159744,"data_stored":43193,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":27654599},"hb_peers":[0,1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":11,"seq":47244640260,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27188,"kb_used_data":152,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344652,"statfs":{"total":96636764160,"available":96608923648,"internally_reserved":0,"allocated":155648,"data_stored":43174,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,1,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":11,"seq":47244640260,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27768,"kb_used_data":736,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344072,"statfs":{"total":96636764160,"available":96608329728,"internally_reserved":0,"allocated":753664,"data_stored":633561,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,2,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":0,"up_from":9,"seq":38654705668,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27760,"kb_used_data":732,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344080,"statfs":{"total":96636764160,"available":96608337920,"internally_reserved":0,"allocated":749568,"data_stored":633542,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[1,2,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T03:39:08.662 INFO:tasks.ceph.ceph_manager.ceph:clean! 2024-09-14T03:39:08.662 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-09-14T03:39:08.662 INFO:tasks.ceph.ceph_manager.ceph:wait_until_healthy 2024-09-14T03:39:08.662 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph health --format=json 2024-09-14T03:39:08.768 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.762+0000 7f8890d11640 1 -- 172.21.15.83:0/126416149 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f888c1318a0 msgr2=0x7f888c133c90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:08.816 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.762+0000 7f8890d11640 1 --2- 172.21.15.83:0/126416149 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f888c1318a0 0x7f888c133c90 secure :-1 s=READY pgs=88 cs=0 l=1 rev1=1 crypto rx=0x7f8884009830 tx=0x7f888402a480 comp rx=0 tx=0).stop 2024-09-14T03:39:08.816 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.763+0000 7f8890d11640 1 -- 172.21.15.83:0/126416149 shutdown_connections 2024-09-14T03:39:08.816 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.763+0000 7f8890d11640 1 --2- 172.21.15.83:0/126416149 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f888c1318a0 0x7f888c133c90 unknown :-1 s=CLOSED pgs=88 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:08.816 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.763+0000 7f8890d11640 1 --2- 172.21.15.83:0/126416149 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f888c0971e0 0x7f888c131360 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:08.816 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.763+0000 7f8890d11640 1 -- 172.21.15.83:0/126416149 >> 172.21.15.83:0/126416149 conn(0x7f888c09c710 msgr2=0x7f888c09cb20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:39:08.817 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.763+0000 7f8890d11640 1 -- 172.21.15.83:0/126416149 shutdown_connections 2024-09-14T03:39:08.817 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.763+0000 7f8890d11640 1 -- 172.21.15.83:0/126416149 wait complete. 2024-09-14T03:39:08.817 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.764+0000 7f8890d11640 1 Processor -- start 2024-09-14T03:39:08.817 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.764+0000 7f8890d11640 1 -- start start 2024-09-14T03:39:08.817 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.764+0000 7f8890d11640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f888c0966f0 0x7f888c06d810 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:39:08.817 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.764+0000 7f8890d11640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f888c0971e0 0x7f888c06dd50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:39:08.817 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.764+0000 7f8890d11640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f888c1318a0 0x7f888c06e290 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:39:08.817 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.764+0000 7f8890d11640 1 -- --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7f888c134a40 con 0x7f888c0971e0 2024-09-14T03:39:08.817 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.764+0000 7f8890d11640 1 -- --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7f888c1348c0 con 0x7f888c0966f0 2024-09-14T03:39:08.817 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.764+0000 7f8890d11640 1 -- --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7f888c134bc0 con 0x7f888c1318a0 2024-09-14T03:39:08.818 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.764+0000 7f888ad9d640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f888c1318a0 0x7f888c06e290 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:39:08.818 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.764+0000 7f8889d9b640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f888c0971e0 0x7f888c06dd50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:39:08.818 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.764+0000 7f888a59c640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f888c0966f0 0x7f888c06d810 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:39:08.818 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.764+0000 7f888ad9d640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f888c1318a0 0x7f888c06e290 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3302/0 says I am v2:172.21.15.83:33110/0 (socket says 172.21.15.83:33110) 2024-09-14T03:39:08.818 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.764+0000 7f888ad9d640 1 -- 172.21.15.83:0/2566999794 learned_addr learned my addr 172.21.15.83:0/2566999794 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:39:08.818 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.764+0000 7f8889d9b640 1 -- 172.21.15.83:0/2566999794 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f888c1318a0 msgr2=0x7f888c06e290 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:08.818 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.764+0000 7f8889d9b640 1 --2- 172.21.15.83:0/2566999794 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f888c1318a0 0x7f888c06e290 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:08.818 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.764+0000 7f8889d9b640 1 -- 172.21.15.83:0/2566999794 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f888c0966f0 msgr2=0x7f888c06d810 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:08.818 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.764+0000 7f8889d9b640 1 --2- 172.21.15.83:0/2566999794 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f888c0966f0 0x7f888c06d810 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:08.818 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.764+0000 7f8889d9b640 1 -- 172.21.15.83:0/2566999794 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f888c06e7d0 con 0x7f888c0971e0 2024-09-14T03:39:08.818 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.765+0000 7f8889d9b640 1 --2- 172.21.15.83:0/2566999794 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f888c0971e0 0x7f888c06dd50 secure :-1 s=READY pgs=89 cs=0 l=1 rev1=1 crypto rx=0x7f8880005fd0 tx=0x7f888000ee50 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:39:08.818 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.765+0000 7f887affd640 1 -- 172.21.15.83:0/2566999794 <== mon.0 v2:172.21.15.83:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f888000cca0 con 0x7f888c0971e0 2024-09-14T03:39:08.818 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.765+0000 7f887affd640 1 -- 172.21.15.83:0/2566999794 <== mon.0 v2:172.21.15.83:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f888000ce40 con 0x7f888c0971e0 2024-09-14T03:39:08.818 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.765+0000 7f8890d11640 1 -- 172.21.15.83:0/2566999794 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f888c08e760 con 0x7f888c0971e0 2024-09-14T03:39:08.819 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.765+0000 7f887affd640 1 -- 172.21.15.83:0/2566999794 <== mon.0 v2:172.21.15.83:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8880004260 con 0x7f888c0971e0 2024-09-14T03:39:08.819 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.765+0000 7f8890d11640 1 -- 172.21.15.83:0/2566999794 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f888c1ca370 con 0x7f888c0971e0 2024-09-14T03:39:08.819 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.766+0000 7f887affd640 1 -- 172.21.15.83:0/2566999794 <== mon.0 v2:172.21.15.83:3300/0 4 ==== mgrmap(e 5) ==== 55999+0+0 (secure 0 0 0) 0x7f88800035f0 con 0x7f888c0971e0 2024-09-14T03:39:08.819 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.766+0000 7f8890d11640 1 -- 172.21.15.83:0/2566999794 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f888c08e590 con 0x7f888c0971e0 2024-09-14T03:39:08.819 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.766+0000 7f887affd640 1 -- 172.21.15.83:0/2566999794 <== mon.0 v2:172.21.15.83:3300/0 5 ==== osd_map(19..19 src has 1..19) ==== 2840+0+0 (secure 0 0 0) 0x7f888005e200 con 0x7f888c0971e0 2024-09-14T03:39:08.820 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.770+0000 7f887affd640 1 -- 172.21.15.83:0/2566999794 <== mon.0 v2:172.21.15.83:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+151260 (secure 0 0 0) 0x7f8880009510 con 0x7f888c0971e0 2024-09-14T03:39:08.937 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.935+0000 7f8890d11640 1 -- 172.21.15.83:0/2566999794 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_command({"prefix": "health", "format": "json"} v 0) -- 0x7f888c06ccc0 con 0x7f888c0971e0 2024-09-14T03:39:08.937 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.935+0000 7f887affd640 1 -- 172.21.15.83:0/2566999794 <== mon.0 v2:172.21.15.83:3300/0 7 ==== mon_command_ack([{"prefix": "health", "format": "json"}]=0 v0) ==== 72+0+46 (secure 0 0 0) 0x7f888003d410 con 0x7f888c0971e0 2024-09-14T03:39:08.937 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:39:08.937 INFO:teuthology.orchestra.run.smithi083.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-09-14T03:39:08.940 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.937+0000 7f8890d11640 1 -- 172.21.15.83:0/2566999794 >> v1:172.21.15.83:6800/1934165865 conn(0x7f886c047fd0 legacy=0x7f886c04a490 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:08.940 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.937+0000 7f8890d11640 1 -- 172.21.15.83:0/2566999794 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f888c0971e0 msgr2=0x7f888c06dd50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:08.940 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.937+0000 7f8890d11640 1 --2- 172.21.15.83:0/2566999794 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f888c0971e0 0x7f888c06dd50 secure :-1 s=READY pgs=89 cs=0 l=1 rev1=1 crypto rx=0x7f8880005fd0 tx=0x7f888000ee50 comp rx=0 tx=0).stop 2024-09-14T03:39:08.940 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.937+0000 7f8890d11640 1 -- 172.21.15.83:0/2566999794 shutdown_connections 2024-09-14T03:39:08.940 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.937+0000 7f8890d11640 1 --2- 172.21.15.83:0/2566999794 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f888c1318a0 0x7f888c06e290 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:08.940 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.937+0000 7f8890d11640 1 --2- 172.21.15.83:0/2566999794 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f888c0971e0 0x7f888c06dd50 unknown :-1 s=CLOSED pgs=89 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:08.941 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.937+0000 7f8890d11640 1 --2- 172.21.15.83:0/2566999794 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f888c0966f0 0x7f888c06d810 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:08.941 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.937+0000 7f8890d11640 1 -- 172.21.15.83:0/2566999794 >> 172.21.15.83:0/2566999794 conn(0x7f888c09c710 msgr2=0x7f888c1339a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:39:08.941 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.937+0000 7f8890d11640 1 -- 172.21.15.83:0/2566999794 shutdown_connections 2024-09-14T03:39:08.941 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:08.937+0000 7f8890d11640 1 -- 172.21.15.83:0/2566999794 wait complete. 2024-09-14T03:39:08.949 INFO:tasks.ceph.ceph_manager.ceph:wait_until_healthy done 2024-09-14T03:39:08.950 INFO:teuthology.run_tasks:Running task exec... 2024-09-14T03:39:08.960 INFO:teuthology.task.exec:Executing custom commands... 2024-09-14T03:39:08.960 INFO:teuthology.task.exec:Running commands on role osd.0 host ubuntu@smithi083.front.sepia.ceph.com 2024-09-14T03:39:08.960 DEBUG:teuthology.orchestra.run.smithi083:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'ceph osd set-require-min-compat-client squid' 2024-09-14T03:39:09.060 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:09.058+0000 7f7938a24640 1 -- 172.21.15.83:0/1622225269 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f793412b4b0 msgr2=0x7f7934132ce0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:09.060 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:09.058+0000 7f7938a24640 1 --2- 172.21.15.83:0/1622225269 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f793412b4b0 0x7f7934132ce0 secure :-1 s=READY pgs=91 cs=0 l=1 rev1=1 crypto rx=0x7f7924009830 tx=0x7f792402a3f0 comp rx=0 tx=0).stop 2024-09-14T03:39:09.060 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:09.058+0000 7f7938a24640 1 -- 172.21.15.83:0/1622225269 shutdown_connections 2024-09-14T03:39:09.061 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:09.058+0000 7f7938a24640 1 --2- 172.21.15.83:0/1622225269 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f793412b4b0 0x7f7934132ce0 unknown :-1 s=CLOSED pgs=91 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:09.061 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:09.058+0000 7f7938a24640 1 --2- 172.21.15.83:0/1622225269 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f79341278b0 0x7f7934127c90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:09.061 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:09.058+0000 7f7938a24640 1 -- 172.21.15.83:0/1622225269 >> 172.21.15.83:0/1622225269 conn(0x7f793409aa00 msgr2=0x7f793409c4d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:39:09.061 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:09.058+0000 7f7938a24640 1 -- 172.21.15.83:0/1622225269 shutdown_connections 2024-09-14T03:39:09.061 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:09.058+0000 7f7938a24640 1 -- 172.21.15.83:0/1622225269 wait complete. 2024-09-14T03:39:09.061 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:09.059+0000 7f7938a24640 1 Processor -- start 2024-09-14T03:39:09.061 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:09.059+0000 7f7938a24640 1 -- start start 2024-09-14T03:39:09.063 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:09.059+0000 7f7938a24640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f79341278b0 0x7f7934130e20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:39:09.063 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:09.059+0000 7f7938a24640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f793412a940 0x7f7934131360 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:39:09.063 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:09.059+0000 7f7938a24640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f793412b4b0 0x7f79341318a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:39:09.063 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:09.059+0000 7f7938a24640 1 -- --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7f7934133820 con 0x7f793412a940 2024-09-14T03:39:09.063 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:09.059+0000 7f7938a24640 1 -- --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7f79341336a0 con 0x7f79341278b0 2024-09-14T03:39:09.063 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:09.059+0000 7f7938a24640 1 -- --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7f79341339a0 con 0x7f793412b4b0 2024-09-14T03:39:09.064 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:09.059+0000 7f7931d9b640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f793412a940 0x7f7934131360 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:39:09.064 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:09.059+0000 7f793259c640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f79341278b0 0x7f7934130e20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:39:09.064 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:09.059+0000 7f7932d9d640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f793412b4b0 0x7f79341318a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:39:09.064 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:09.059+0000 7f793259c640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f79341278b0 0x7f7934130e20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3301/0 says I am v2:172.21.15.83:60820/0 (socket says 172.21.15.83:60820) 2024-09-14T03:39:09.064 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:09.059+0000 7f793259c640 1 -- 172.21.15.83:0/3220401337 learned_addr learned my addr 172.21.15.83:0/3220401337 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:39:09.064 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:09.059+0000 7f7931d9b640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f793412a940 0x7f7934131360 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3300/0 says I am v2:172.21.15.83:36124/0 (socket says 172.21.15.83:36124) 2024-09-14T03:39:09.064 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:09.059+0000 7f7932d9d640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f793412b4b0 0x7f79341318a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3302/0 says I am v2:172.21.15.83:33122/0 (socket says 172.21.15.83:33122) 2024-09-14T03:39:09.064 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:09.060+0000 7f793259c640 1 -- 172.21.15.83:0/3220401337 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f793412b4b0 msgr2=0x7f79341318a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:09.064 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:09.060+0000 7f793259c640 1 --2- 172.21.15.83:0/3220401337 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f793412b4b0 0x7f79341318a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:09.064 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:09.060+0000 7f793259c640 1 -- 172.21.15.83:0/3220401337 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f793412a940 msgr2=0x7f7934131360 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:09.064 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:09.060+0000 7f793259c640 1 --2- 172.21.15.83:0/3220401337 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f793412a940 0x7f7934131360 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:09.064 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:09.060+0000 7f793259c640 1 -- 172.21.15.83:0/3220401337 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f7934070ed0 con 0x7f79341278b0 2024-09-14T03:39:09.064 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:09.060+0000 7f7931d9b640 1 --2- 172.21.15.83:0/3220401337 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f793412a940 0x7f7934131360 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:39:09.065 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:09.060+0000 7f793259c640 1 --2- 172.21.15.83:0/3220401337 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f79341278b0 0x7f7934130e20 secure :-1 s=READY pgs=49 cs=0 l=1 rev1=1 crypto rx=0x7f792c009d50 tx=0x7f792c00c6a0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:39:09.065 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:09.060+0000 7f792affd640 1 -- 172.21.15.83:0/3220401337 <== mon.1 v2:172.21.15.83:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f792c003b70 con 0x7f79341278b0 2024-09-14T03:39:09.065 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:09.060+0000 7f792affd640 1 -- 172.21.15.83:0/3220401337 <== mon.1 v2:172.21.15.83:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f792c003d10 con 0x7f79341278b0 2024-09-14T03:39:09.065 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:09.060+0000 7f792affd640 1 -- 172.21.15.83:0/3220401337 <== mon.1 v2:172.21.15.83:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f792c004280 con 0x7f79341278b0 2024-09-14T03:39:09.065 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:09.060+0000 7f7938a24640 1 -- 172.21.15.83:0/3220401337 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f7934113cf0 con 0x7f79341278b0 2024-09-14T03:39:09.065 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:09.060+0000 7f7938a24640 1 -- 172.21.15.83:0/3220401337 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7f7934132980 con 0x7f79341278b0 2024-09-14T03:39:09.065 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:09.061+0000 7f7938a24640 1 -- 172.21.15.83:0/3220401337 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f7934127c90 con 0x7f79341278b0 2024-09-14T03:39:09.065 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:09.061+0000 7f792affd640 1 -- 172.21.15.83:0/3220401337 <== mon.1 v2:172.21.15.83:3301/0 4 ==== mgrmap(e 5) ==== 55999+0+0 (secure 0 0 0) 0x7f792c004420 con 0x7f79341278b0 2024-09-14T03:39:09.065 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:09.061+0000 7f792affd640 1 -- 172.21.15.83:0/3220401337 <== mon.1 v2:172.21.15.83:3301/0 5 ==== osd_map(19..19 src has 1..19) ==== 2840+0+0 (secure 0 0 0) 0x7f792c05e620 con 0x7f79341278b0 2024-09-14T03:39:09.066 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:09.065+0000 7f792affd640 1 -- 172.21.15.83:0/3220401337 <== mon.1 v2:172.21.15.83:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+151260 (secure 0 0 0) 0x7f792c05d790 con 0x7f79341278b0 2024-09-14T03:39:09.213 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:09.212+0000 7f7938a24640 1 -- 172.21.15.83:0/3220401337 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_command({"prefix": "osd set-require-min-compat-client", "version": "squid"} v 0) -- 0x7f7934075ab0 con 0x7f79341278b0 2024-09-14T03:39:09.828 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:09.827+0000 7f792affd640 1 -- 172.21.15.83:0/3220401337 <== mon.1 v2:172.21.15.83:3301/0 7 ==== mon_command_ack([{"prefix": "osd set-require-min-compat-client", "version": "squid"}]=0 set require_min_compat_client to squid v20) ==== 139+0+0 (secure 0 0 0) 0x7f792c03d870 con 0x7f79341278b0 2024-09-14T03:39:09.828 INFO:teuthology.orchestra.run.smithi083.stderr:set require_min_compat_client to squid 2024-09-14T03:39:09.831 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:09.829+0000 7f7938a24640 1 -- 172.21.15.83:0/3220401337 >> v1:172.21.15.83:6800/1934165865 conn(0x7f7914047fd0 legacy=0x7f791404a490 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:09.831 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:09.829+0000 7f7938a24640 1 -- 172.21.15.83:0/3220401337 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f79341278b0 msgr2=0x7f7934130e20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:09.831 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:09.829+0000 7f7938a24640 1 --2- 172.21.15.83:0/3220401337 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f79341278b0 0x7f7934130e20 secure :-1 s=READY pgs=49 cs=0 l=1 rev1=1 crypto rx=0x7f792c009d50 tx=0x7f792c00c6a0 comp rx=0 tx=0).stop 2024-09-14T03:39:09.831 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:09.829+0000 7f7938a24640 1 -- 172.21.15.83:0/3220401337 shutdown_connections 2024-09-14T03:39:09.831 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:09.829+0000 7f7938a24640 1 --2- 172.21.15.83:0/3220401337 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f793412b4b0 0x7f79341318a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:09.831 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:09.829+0000 7f7938a24640 1 --2- 172.21.15.83:0/3220401337 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f793412a940 0x7f7934131360 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:09.831 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:09.829+0000 7f7938a24640 1 --2- 172.21.15.83:0/3220401337 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f79341278b0 0x7f7934130e20 unknown :-1 s=CLOSED pgs=49 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:09.831 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:09.829+0000 7f7938a24640 1 -- 172.21.15.83:0/3220401337 >> 172.21.15.83:0/3220401337 conn(0x7f793409aa00 msgr2=0x7f793406fc70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:39:09.831 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:09.829+0000 7f7938a24640 1 -- 172.21.15.83:0/3220401337 shutdown_connections 2024-09-14T03:39:09.831 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:09.829+0000 7f7938a24640 1 -- 172.21.15.83:0/3220401337 wait complete. 2024-09-14T03:39:09.844 INFO:teuthology.run_tasks:Running task print... 2024-09-14T03:39:09.853 INFO:teuthology.task.print:**** done ceph 2024-09-14T03:39:09.853 INFO:teuthology.run_tasks:Running task print... 2024-09-14T03:39:09.884 INFO:teuthology.task.print:*** upgrading, no cephfs present 2024-09-14T03:39:09.884 INFO:teuthology.run_tasks:Running task exec... 2024-09-14T03:39:09.918 INFO:teuthology.task.exec:Executing custom commands... 2024-09-14T03:39:09.918 INFO:teuthology.task.exec:Running commands on role mon.a host ubuntu@smithi083.front.sepia.ceph.com 2024-09-14T03:39:09.919 DEBUG:teuthology.orchestra.run.smithi083:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'ceph fs dump' 2024-09-14T03:39:10.025 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:10.024+0000 7f6178776640 1 -- 172.21.15.83:0/2608540365 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f6170070d50 msgr2=0x7f61700711e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:10.026 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:10.024+0000 7f6178776640 1 --2- 172.21.15.83:0/2608540365 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f6170070d50 0x7f61700711e0 secure :-1 s=READY pgs=52 cs=0 l=1 rev1=1 crypto rx=0x7f6160005fd0 tx=0x7f616002a770 comp rx=0 tx=0).stop 2024-09-14T03:39:10.026 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:10.024+0000 7f6178776640 1 -- 172.21.15.83:0/2608540365 shutdown_connections 2024-09-14T03:39:10.026 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:10.024+0000 7f6178776640 1 --2- 172.21.15.83:0/2608540365 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f6170072bb0 0x7f6170070810 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:10.026 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:10.024+0000 7f6178776640 1 --2- 172.21.15.83:0/2608540365 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f6170070d50 0x7f61700711e0 unknown :-1 s=CLOSED pgs=52 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:10.026 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:10.024+0000 7f6178776640 1 --2- 172.21.15.83:0/2608540365 >> v2:172.21.15.83:3300/0 conn(0x7f617012abd0 0x7f617012afb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:10.026 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:10.024+0000 7f6178776640 1 -- 172.21.15.83:0/2608540365 >> 172.21.15.83:0/2608540365 conn(0x7f617009ef70 msgr2=0x7f617009f380 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:39:10.026 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:10.024+0000 7f6178776640 1 -- 172.21.15.83:0/2608540365 shutdown_connections 2024-09-14T03:39:10.027 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:10.024+0000 7f6178776640 1 -- 172.21.15.83:0/2608540365 wait complete. 2024-09-14T03:39:10.029 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:10.025+0000 7f6178776640 1 Processor -- start 2024-09-14T03:39:10.029 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:10.025+0000 7f6178776640 1 -- start start 2024-09-14T03:39:10.029 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:10.025+0000 7f6178776640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f6170070d50 0x7f6170131880 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:39:10.029 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:10.025+0000 7f6178776640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f6170072bb0 0x7f6170131dc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:39:10.029 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:10.025+0000 7f6178776640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f617012abd0 0x7f6170132300 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:39:10.029 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:10.025+0000 7f6178776640 1 -- --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7f61701359d0 con 0x7f6170070d50 2024-09-14T03:39:10.030 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:10.025+0000 7f6178776640 1 -- --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7f6170135850 con 0x7f6170072bb0 2024-09-14T03:39:10.030 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:10.025+0000 7f6178776640 1 -- --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7f6170135b50 con 0x7f617012abd0 2024-09-14T03:39:10.030 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:10.025+0000 7f6176d13640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f617012abd0 0x7f6170132300 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:39:10.030 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:10.025+0000 7f6176512640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f6170070d50 0x7f6170131880 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:39:10.030 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:10.025+0000 7f6175d11640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f6170072bb0 0x7f6170131dc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:39:10.030 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:10.025+0000 7f6176d13640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f617012abd0 0x7f6170132300 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3302/0 says I am v2:172.21.15.83:33136/0 (socket says 172.21.15.83:33136) 2024-09-14T03:39:10.030 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:10.025+0000 7f6175d11640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f6170072bb0 0x7f6170131dc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3301/0 says I am v2:172.21.15.83:60836/0 (socket says 172.21.15.83:60836) 2024-09-14T03:39:10.030 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:10.025+0000 7f6176512640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f6170070d50 0x7f6170131880 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3300/0 says I am v2:172.21.15.83:36140/0 (socket says 172.21.15.83:36140) 2024-09-14T03:39:10.030 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:10.025+0000 7f6176d13640 1 -- 172.21.15.83:0/1332502316 learned_addr learned my addr 172.21.15.83:0/1332502316 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:39:10.030 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:10.026+0000 7f6176d13640 1 -- 172.21.15.83:0/1332502316 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f6170072bb0 msgr2=0x7f6170131dc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:10.030 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:10.026+0000 7f6176d13640 1 --2- 172.21.15.83:0/1332502316 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f6170072bb0 0x7f6170131dc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:10.030 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:10.026+0000 7f6176d13640 1 -- 172.21.15.83:0/1332502316 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f6170070d50 msgr2=0x7f6170131880 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:10.030 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:10.026+0000 7f6176d13640 1 --2- 172.21.15.83:0/1332502316 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f6170070d50 0x7f6170131880 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:10.031 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:10.026+0000 7f6176d13640 1 -- 172.21.15.83:0/1332502316 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f6170133170 con 0x7f617012abd0 2024-09-14T03:39:10.031 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:10.026+0000 7f6176d13640 1 --2- 172.21.15.83:0/1332502316 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f617012abd0 0x7f6170132300 secure :-1 s=READY pgs=53 cs=0 l=1 rev1=1 crypto rx=0x7f6160034040 tx=0x7f6160032de0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:39:10.031 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:10.026+0000 7f616effd640 1 -- 172.21.15.83:0/1332502316 <== mon.2 v2:172.21.15.83:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6160041020 con 0x7f617012abd0 2024-09-14T03:39:10.031 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:10.026+0000 7f616effd640 1 -- 172.21.15.83:0/1332502316 <== mon.2 v2:172.21.15.83:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f6160004040 con 0x7f617012abd0 2024-09-14T03:39:10.031 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:10.026+0000 7f6178776640 1 -- 172.21.15.83:0/1332502316 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f6170132840 con 0x7f617012abd0 2024-09-14T03:39:10.031 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:10.026+0000 7f616effd640 1 -- 172.21.15.83:0/1332502316 <== mon.2 v2:172.21.15.83:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6160036040 con 0x7f617012abd0 2024-09-14T03:39:10.031 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:10.026+0000 7f6178776640 1 -- 172.21.15.83:0/1332502316 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7f6170134170 con 0x7f617012abd0 2024-09-14T03:39:10.031 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:10.027+0000 7f6178776640 1 -- 172.21.15.83:0/1332502316 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f6170070810 con 0x7f617012abd0 2024-09-14T03:39:10.031 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:10.027+0000 7f616effd640 1 -- 172.21.15.83:0/1332502316 <== mon.2 v2:172.21.15.83:3302/0 4 ==== mgrmap(e 5) ==== 55999+0+0 (secure 0 0 0) 0x7f6160044020 con 0x7f617012abd0 2024-09-14T03:39:10.031 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:10.028+0000 7f616effd640 1 -- 172.21.15.83:0/1332502316 <== mon.2 v2:172.21.15.83:3302/0 5 ==== osd_map(20..20 src has 1..20) ==== 2812+0+0 (secure 0 0 0) 0x7f616007e370 con 0x7f617012abd0 2024-09-14T03:39:10.033 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:10.031+0000 7f616effd640 1 -- 172.21.15.83:0/1332502316 <== mon.2 v2:172.21.15.83:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+151260 (secure 0 0 0) 0x7f616004b070 con 0x7f617012abd0 2024-09-14T03:39:10.194 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:10.192+0000 7f6178776640 1 -- 172.21.15.83:0/1332502316 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_command({"prefix": "fs dump"} v 0) -- 0x7f617012d100 con 0x7f617012abd0 2024-09-14T03:39:10.194 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:10.192+0000 7f616effd640 1 -- 172.21.15.83:0/1332502316 <== mon.2 v2:172.21.15.83:3302/0 7 ==== mon_command_ack([{"prefix": "fs dump"}]=0 dumped fsmap epoch 1 v1) ==== 75+0+440 (secure 0 0 0) 0x7f616007d500 con 0x7f617012abd0 2024-09-14T03:39:10.194 INFO:teuthology.orchestra.run.smithi083.stdout:e1 2024-09-14T03:39:10.194 INFO:teuthology.orchestra.run.smithi083.stdout:btime 2024-09-14T03:38:43:953582+0000 2024-09-14T03:39:10.194 INFO:teuthology.orchestra.run.smithi083.stdout:enable_multiple, ever_enabled_multiple: 1,1 2024-09-14T03:39:10.194 INFO:teuthology.orchestra.run.smithi083.stdout:default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2,11=minor log segments,12=quiesce subvolumes} 2024-09-14T03:39:10.195 INFO:teuthology.orchestra.run.smithi083.stdout:legacy client fscid: -1 2024-09-14T03:39:10.195 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:39:10.195 INFO:teuthology.orchestra.run.smithi083.stdout:No filesystems configured 2024-09-14T03:39:10.195 INFO:teuthology.orchestra.run.smithi083.stderr:dumped fsmap epoch 1 2024-09-14T03:39:10.196 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:10.194+0000 7f6178776640 1 -- 172.21.15.83:0/1332502316 >> v1:172.21.15.83:6800/1934165865 conn(0x7f615c047fd0 legacy=0x7f615c04a490 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:10.196 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:10.194+0000 7f6178776640 1 -- 172.21.15.83:0/1332502316 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f617012abd0 msgr2=0x7f6170132300 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:39:10.196 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:10.194+0000 7f6178776640 1 --2- 172.21.15.83:0/1332502316 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f617012abd0 0x7f6170132300 secure :-1 s=READY pgs=53 cs=0 l=1 rev1=1 crypto rx=0x7f6160034040 tx=0x7f6160032de0 comp rx=0 tx=0).stop 2024-09-14T03:39:10.196 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:10.194+0000 7f6178776640 1 -- 172.21.15.83:0/1332502316 shutdown_connections 2024-09-14T03:39:10.196 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:10.194+0000 7f6178776640 1 --2- 172.21.15.83:0/1332502316 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f617012abd0 0x7f6170132300 unknown :-1 s=CLOSED pgs=53 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:10.196 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:10.194+0000 7f6178776640 1 --2- 172.21.15.83:0/1332502316 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f6170072bb0 0x7f6170131dc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:10.196 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:10.194+0000 7f6178776640 1 --2- 172.21.15.83:0/1332502316 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f6170070d50 0x7f6170131880 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:39:10.196 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:10.194+0000 7f6178776640 1 -- 172.21.15.83:0/1332502316 >> 172.21.15.83:0/1332502316 conn(0x7f617009ef70 msgr2=0x7f61700700c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:39:10.196 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:10.194+0000 7f6178776640 1 -- 172.21.15.83:0/1332502316 shutdown_connections 2024-09-14T03:39:10.197 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:39:10.194+0000 7f6178776640 1 -- 172.21.15.83:0/1332502316 wait complete. 2024-09-14T03:39:10.206 INFO:teuthology.run_tasks:Running task install.upgrade... 2024-09-14T03:39:10.214 INFO:teuthology.task.install:project ceph config {'mon.a': None} overrides {'flavor': 'default', 'sha1': 'a2782315255a3e7d5f6cdc9eb35785187ff6ddf8'} 2024-09-14T03:39:10.215 INFO:teuthology.task.install:remote ubuntu@smithi083.front.sepia.ceph.com config {'flavor': 'default', 'sha1': 'a2782315255a3e7d5f6cdc9eb35785187ff6ddf8'} 2024-09-14T03:39:10.215 INFO:teuthology.task.install:extra packages: [] 2024-09-14T03:39:10.220 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'cephadm', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'ceph-volume', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'cephadm', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'ceph-fuse', 'ceph-volume', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2024-09-14T03:39:10.220 INFO:teuthology.task.install:Upgrading ceph rpm packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, ceph-volume, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd 2024-09-14T03:39:10.220 DEBUG:teuthology.orchestra.run.smithi083:> rpm -q ceph-common --qf '%{VERSION}-%{RELEASE}' 2024-09-14T03:39:10.242 INFO:teuthology.orchestra.run.smithi083.stdout:19.1.1-324.gecab7be5.el9 2024-09-14T03:39:10.242 INFO:teuthology.packaging:The installed version of ceph-common is 19.1.1-324.gecab7be5.el9 2024-09-14T03:39:10.242 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&sha1=a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T03:39:10.385 INFO:teuthology.task.install:Ceph rpm upgrade from 19.1.1-324.gecab7be5.el9 to 19.3.0-4907.ga2782315 2024-09-14T03:39:10.385 INFO:teuthology.task.install.rpm:Host smithi083 is: centos 9.stream x86_64 2024-09-14T03:39:10.386 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&sha1=a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T03:39:10.527 INFO:teuthology.task.install.rpm:Repo base URL: https://4.chacra.ceph.com/r/ceph/main/a2782315255a3e7d5f6cdc9eb35785187ff6ddf8/centos/9/flavors/default/ 2024-09-14T03:39:10.527 DEBUG:teuthology.orchestra.run.smithi083:> sudo rm -f /etc/yum.repos.d/ceph.repo 2024-09-14T03:39:10.775 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://4.chacra.ceph.com/r/ceph/main/a2782315255a3e7d5f6cdc9eb35785187ff6ddf8/centos/9/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://4.chacra.ceph.com/r/ceph/main/a2782315255a3e7d5f6cdc9eb35785187ff6ddf8/centos/9/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://4.chacra.ceph.com/r/ceph/main/a2782315255a3e7d5f6cdc9eb35785187ff6ddf8/centos/9/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2024-09-14T03:39:10.775 DEBUG:teuthology.orchestra.run.smithi083:> set -ex 2024-09-14T03:39:10.775 DEBUG:teuthology.orchestra.run.smithi083:> sudo dd of=/etc/yum.repos.d/ceph.repo 2024-09-14T03:39:10.801 DEBUG:teuthology.orchestra.run.smithi083:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;sha1/a2782315255a3e7d5f6cdc9eb35785187ff6ddf8/;g' /etc/yum.repos.d/ceph.repo ; fi 2024-09-14T03:39:10.867 DEBUG:teuthology.orchestra.run.smithi083:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2024-09-14T03:39:10.933 DEBUG:teuthology.orchestra.run.smithi083:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2024-09-14T03:39:10.988 INFO:teuthology.orchestra.run.smithi083.stdout:check_obsoletes = 1 2024-09-14T03:39:10.999 DEBUG:teuthology.orchestra.run.smithi083:> sudo yum clean all 2024-09-14T03:39:11.229 INFO:teuthology.orchestra.run.smithi083.stdout:60 files removed 2024-09-14T03:39:11.257 DEBUG:teuthology.orchestra.run.smithi083:> sudo yum -y upgrade ceph-radosgw ceph-test ceph ceph-base cephadm ceph-immutable-object-cache ceph-mgr ceph-mgr-dashboard ceph-mgr-diskprediction-local ceph-mgr-rook ceph-mgr-cephadm ceph-fuse ceph-volume librados-devel libcephfs2 libcephfs-devel librados2 librbd1 python3-rados python3-rgw python3-cephfs python3-rbd rbd-fuse rbd-mirror rbd-nbd 2024-09-14T03:39:12.089 INFO:teuthology.orchestra.run.smithi083.stdout:ceph packages for x86_64 156 kB/s | 84 kB 00:00 2024-09-14T03:39:12.614 INFO:teuthology.orchestra.run.smithi083.stdout:ceph noarch packages 50 kB/s | 24 kB 00:00 2024-09-14T03:39:13.115 INFO:teuthology.orchestra.run.smithi083.stdout:ceph source packages 4.2 kB/s | 1.9 kB 00:00 2024-09-14T03:39:13.806 INFO:teuthology.orchestra.run.smithi083.stdout:CentOS Stream 9 - BaseOS 13 MB/s | 8.2 MB 00:00 2024-09-14T03:39:17.275 INFO:teuthology.orchestra.run.smithi083.stdout:CentOS Stream 9 - AppStream 8.6 MB/s | 20 MB 00:02 2024-09-14T03:39:23.392 INFO:teuthology.orchestra.run.smithi083.stdout:CentOS Stream 9 - CRB 2.9 MB/s | 6.5 MB 00:02 2024-09-14T03:39:24.892 INFO:teuthology.orchestra.run.smithi083.stdout:CentOS Stream 9 - Extras packages 66 kB/s | 19 kB 00:00 2024-09-14T03:39:25.964 INFO:teuthology.orchestra.run.smithi083.stdout:Extra Packages for Enterprise Linux 24 MB/s | 23 MB 00:00 2024-09-14T03:39:31.445 INFO:teuthology.orchestra.run.smithi083.stdout:lab-extras 37 kB/s | 1.7 kB 00:00 2024-09-14T03:39:33.747 INFO:teuthology.orchestra.run.smithi083.stderr:Package cephadm available, but not installed. 2024-09-14T03:39:33.748 INFO:teuthology.orchestra.run.smithi083.stdout:No match for argument: cephadm 2024-09-14T03:39:33.754 INFO:teuthology.orchestra.run.smithi083.stderr:Package ceph-mgr-dashboard available, but not installed. 2024-09-14T03:39:33.754 INFO:teuthology.orchestra.run.smithi083.stdout:No match for argument: ceph-mgr-dashboard 2024-09-14T03:39:33.754 INFO:teuthology.orchestra.run.smithi083.stderr:Package ceph-mgr-diskprediction-local available, but not installed. 2024-09-14T03:39:33.755 INFO:teuthology.orchestra.run.smithi083.stdout:No match for argument: ceph-mgr-diskprediction-local 2024-09-14T03:39:33.755 INFO:teuthology.orchestra.run.smithi083.stderr:Package ceph-mgr-rook available, but not installed. 2024-09-14T03:39:33.755 INFO:teuthology.orchestra.run.smithi083.stdout:No match for argument: ceph-mgr-rook 2024-09-14T03:39:33.756 INFO:teuthology.orchestra.run.smithi083.stderr:Package ceph-mgr-cephadm available, but not installed. 2024-09-14T03:39:33.756 INFO:teuthology.orchestra.run.smithi083.stdout:No match for argument: ceph-mgr-cephadm 2024-09-14T03:39:33.759 INFO:teuthology.orchestra.run.smithi083.stderr:Package ceph-volume available, but not installed. 2024-09-14T03:39:33.759 INFO:teuthology.orchestra.run.smithi083.stdout:No match for argument: ceph-volume 2024-09-14T03:39:33.848 INFO:teuthology.orchestra.run.smithi083.stdout:Dependencies resolved. 2024-09-14T03:39:33.851 INFO:teuthology.orchestra.run.smithi083.stdout:================================================================================ 2024-09-14T03:39:33.851 INFO:teuthology.orchestra.run.smithi083.stdout: Package Arch Version Repository Size 2024-09-14T03:39:33.851 INFO:teuthology.orchestra.run.smithi083.stdout:================================================================================ 2024-09-14T03:39:33.851 INFO:teuthology.orchestra.run.smithi083.stdout:Upgrading: 2024-09-14T03:39:33.851 INFO:teuthology.orchestra.run.smithi083.stdout: ceph x86_64 2:19.3.0-4907.ga2782315.el9 ceph 6.6 k 2024-09-14T03:39:33.851 INFO:teuthology.orchestra.run.smithi083.stdout: ceph-base x86_64 2:19.3.0-4907.ga2782315.el9 ceph 5.5 M 2024-09-14T03:39:33.852 INFO:teuthology.orchestra.run.smithi083.stdout: ceph-common x86_64 2:19.3.0-4907.ga2782315.el9 ceph 22 M 2024-09-14T03:39:33.852 INFO:teuthology.orchestra.run.smithi083.stdout: ceph-fuse x86_64 2:19.3.0-4907.ga2782315.el9 ceph 900 k 2024-09-14T03:39:33.852 INFO:teuthology.orchestra.run.smithi083.stdout: ceph-immutable-object-cache 2024-09-14T03:39:33.852 INFO:teuthology.orchestra.run.smithi083.stdout: x86_64 2:19.3.0-4907.ga2782315.el9 ceph 145 k 2024-09-14T03:39:33.852 INFO:teuthology.orchestra.run.smithi083.stdout: ceph-mds x86_64 2:19.3.0-4907.ga2782315.el9 ceph 2.4 M 2024-09-14T03:39:33.852 INFO:teuthology.orchestra.run.smithi083.stdout: ceph-mgr x86_64 2:19.3.0-4907.ga2782315.el9 ceph 1.5 M 2024-09-14T03:39:33.852 INFO:teuthology.orchestra.run.smithi083.stdout: ceph-mgr-modules-core noarch 2:19.3.0-4907.ga2782315.el9 ceph-noarch 291 k 2024-09-14T03:39:33.852 INFO:teuthology.orchestra.run.smithi083.stdout: ceph-mon x86_64 2:19.3.0-4907.ga2782315.el9 ceph 4.8 M 2024-09-14T03:39:33.852 INFO:teuthology.orchestra.run.smithi083.stdout: ceph-osd x86_64 2:19.3.0-4907.ga2782315.el9 ceph 17 M 2024-09-14T03:39:33.852 INFO:teuthology.orchestra.run.smithi083.stdout: ceph-radosgw x86_64 2:19.3.0-4907.ga2782315.el9 ceph 11 M 2024-09-14T03:39:33.853 INFO:teuthology.orchestra.run.smithi083.stdout: ceph-selinux x86_64 2:19.3.0-4907.ga2782315.el9 ceph 25 k 2024-09-14T03:39:33.853 INFO:teuthology.orchestra.run.smithi083.stdout: ceph-test x86_64 2:19.3.0-4907.ga2782315.el9 ceph 50 M 2024-09-14T03:39:33.853 INFO:teuthology.orchestra.run.smithi083.stdout: libcephfs-devel x86_64 2:19.3.0-4907.ga2782315.el9 ceph 32 k 2024-09-14T03:39:33.853 INFO:teuthology.orchestra.run.smithi083.stdout: libcephfs2 x86_64 2:19.3.0-4907.ga2782315.el9 ceph 756 k 2024-09-14T03:39:33.853 INFO:teuthology.orchestra.run.smithi083.stdout: libcephsqlite x86_64 2:19.3.0-4907.ga2782315.el9 ceph 161 k 2024-09-14T03:39:33.853 INFO:teuthology.orchestra.run.smithi083.stdout: librados-devel x86_64 2:19.3.0-4907.ga2782315.el9 ceph 127 k 2024-09-14T03:39:33.853 INFO:teuthology.orchestra.run.smithi083.stdout: librados2 x86_64 2:19.3.0-4907.ga2782315.el9 ceph 3.5 M 2024-09-14T03:39:33.853 INFO:teuthology.orchestra.run.smithi083.stdout: libradosstriper1 x86_64 2:19.3.0-4907.ga2782315.el9 ceph 508 k 2024-09-14T03:39:33.853 INFO:teuthology.orchestra.run.smithi083.stdout: librbd1 x86_64 2:19.3.0-4907.ga2782315.el9 ceph 3.2 M 2024-09-14T03:39:33.853 INFO:teuthology.orchestra.run.smithi083.stdout: librgw2 x86_64 2:19.3.0-4907.ga2782315.el9 ceph 5.5 M 2024-09-14T03:39:33.853 INFO:teuthology.orchestra.run.smithi083.stdout: python3-ceph-argparse x86_64 2:19.3.0-4907.ga2782315.el9 ceph 45 k 2024-09-14T03:39:33.853 INFO:teuthology.orchestra.run.smithi083.stdout: python3-ceph-common x86_64 2:19.3.0-4907.ga2782315.el9 ceph 143 k 2024-09-14T03:39:33.853 INFO:teuthology.orchestra.run.smithi083.stdout: python3-cephfs x86_64 2:19.3.0-4907.ga2782315.el9 ceph 161 k 2024-09-14T03:39:33.853 INFO:teuthology.orchestra.run.smithi083.stdout: python3-rados x86_64 2:19.3.0-4907.ga2782315.el9 ceph 320 k 2024-09-14T03:39:33.854 INFO:teuthology.orchestra.run.smithi083.stdout: python3-rbd x86_64 2:19.3.0-4907.ga2782315.el9 ceph 305 k 2024-09-14T03:39:33.854 INFO:teuthology.orchestra.run.smithi083.stdout: python3-rgw x86_64 2:19.3.0-4907.ga2782315.el9 ceph 99 k 2024-09-14T03:39:33.854 INFO:teuthology.orchestra.run.smithi083.stdout: rbd-fuse x86_64 2:19.3.0-4907.ga2782315.el9 ceph 85 k 2024-09-14T03:39:33.854 INFO:teuthology.orchestra.run.smithi083.stdout: rbd-mirror x86_64 2:19.3.0-4907.ga2782315.el9 ceph 3.2 M 2024-09-14T03:39:33.854 INFO:teuthology.orchestra.run.smithi083.stdout: rbd-nbd x86_64 2:19.3.0-4907.ga2782315.el9 ceph 171 k 2024-09-14T03:39:33.854 INFO:teuthology.orchestra.run.smithi083.stdout:Installing dependencies: 2024-09-14T03:39:33.854 INFO:teuthology.orchestra.run.smithi083.stdout: libnbd x86_64 1.20.2-2.el9 appstream 171 k 2024-09-14T03:39:33.854 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:39:33.854 INFO:teuthology.orchestra.run.smithi083.stdout:Transaction Summary 2024-09-14T03:39:33.854 INFO:teuthology.orchestra.run.smithi083.stdout:================================================================================ 2024-09-14T03:39:33.854 INFO:teuthology.orchestra.run.smithi083.stdout:Install 1 Package 2024-09-14T03:39:33.854 INFO:teuthology.orchestra.run.smithi083.stdout:Upgrade 30 Packages 2024-09-14T03:39:33.854 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:39:33.855 INFO:teuthology.orchestra.run.smithi083.stdout:Total download size: 134 M 2024-09-14T03:39:33.855 INFO:teuthology.orchestra.run.smithi083.stdout:Downloading Packages: 2024-09-14T03:39:34.064 INFO:teuthology.orchestra.run.smithi083.stdout:(1/31): libnbd-1.20.2-2.el9.x86_64.rpm 1.5 MB/s | 171 kB 00:00 2024-09-14T03:39:34.089 INFO:teuthology.orchestra.run.smithi083.stdout:(2/31): ceph-19.3.0-4907.ga2782315.el9.x86_64.r 49 kB/s | 6.6 kB 00:00 2024-09-14T03:39:34.422 INFO:teuthology.orchestra.run.smithi083.stdout:(3/31): ceph-base-19.3.0-4907.ga2782315.el9.x86 12 MB/s | 5.5 MB 00:00 2024-09-14T03:39:34.481 INFO:teuthology.orchestra.run.smithi083.stdout:(4/31): ceph-fuse-19.3.0-4907.ga2782315.el9.x86 2.2 MB/s | 900 kB 00:00 2024-09-14T03:39:34.506 INFO:teuthology.orchestra.run.smithi083.stdout:(5/31): ceph-immutable-object-cache-19.3.0-4907 1.7 MB/s | 145 kB 00:00 2024-09-14T03:39:34.598 INFO:teuthology.orchestra.run.smithi083.stdout:(6/31): ceph-mgr-19.3.0-4907.ga2782315.el9.x86_ 17 MB/s | 1.5 MB 00:00 2024-09-14T03:39:34.839 INFO:teuthology.orchestra.run.smithi083.stdout:(7/31): ceph-mon-19.3.0-4907.ga2782315.el9.x86_ 20 MB/s | 4.8 MB 00:00 2024-09-14T03:39:35.082 INFO:teuthology.orchestra.run.smithi083.stdout:(8/31): ceph-mds-19.3.0-4907.ga2782315.el9.x86_ 4.0 MB/s | 2.4 MB 00:00 2024-09-14T03:39:35.408 INFO:teuthology.orchestra.run.smithi083.stdout:(9/31): ceph-common-19.3.0-4907.ga2782315.el9.x 16 MB/s | 22 MB 00:01 2024-09-14T03:39:35.575 INFO:teuthology.orchestra.run.smithi083.stdout:(10/31): ceph-osd-19.3.0-4907.ga2782315.el9.x86 23 MB/s | 17 MB 00:00 2024-09-14T03:39:35.591 INFO:teuthology.orchestra.run.smithi083.stdout:(11/31): ceph-selinux-19.3.0-4907.ga2782315.el9 137 kB/s | 25 kB 00:00 2024-09-14T03:39:35.642 INFO:teuthology.orchestra.run.smithi083.stdout:(12/31): libcephfs-devel-19.3.0-4907.ga2782315. 635 kB/s | 32 kB 00:00 2024-09-14T03:39:35.784 INFO:teuthology.orchestra.run.smithi083.stdout:(13/31): libcephfs2-19.3.0-4907.ga2782315.el9.x 5.2 MB/s | 756 kB 00:00 2024-09-14T03:39:35.851 INFO:teuthology.orchestra.run.smithi083.stdout:(14/31): libcephsqlite-19.3.0-4907.ga2782315.el 2.4 MB/s | 161 kB 00:00 2024-09-14T03:39:35.934 INFO:teuthology.orchestra.run.smithi083.stdout:(15/31): librados-devel-19.3.0-4907.ga2782315.e 1.5 MB/s | 127 kB 00:00 2024-09-14T03:39:36.185 INFO:teuthology.orchestra.run.smithi083.stdout:(16/31): librados2-19.3.0-4907.ga2782315.el9.x8 14 MB/s | 3.5 MB 00:00 2024-09-14T03:39:36.310 INFO:teuthology.orchestra.run.smithi083.stdout:(17/31): ceph-radosgw-19.3.0-4907.ga2782315.el9 8.9 MB/s | 11 MB 00:01 2024-09-14T03:39:36.335 INFO:teuthology.orchestra.run.smithi083.stdout:(18/31): libradosstriper1-19.3.0-4907.ga2782315 3.3 MB/s | 508 kB 00:00 2024-09-14T03:39:36.619 INFO:teuthology.orchestra.run.smithi083.stdout:(19/31): librbd1-19.3.0-4907.ga2782315.el9.x86_ 11 MB/s | 3.2 MB 00:00 2024-09-14T03:39:36.694 INFO:teuthology.orchestra.run.smithi083.stdout:(20/31): librgw2-19.3.0-4907.ga2782315.el9.x86_ 15 MB/s | 5.5 MB 00:00 2024-09-14T03:39:36.719 INFO:teuthology.orchestra.run.smithi083.stdout:(21/31): python3-ceph-argparse-19.3.0-4907.ga27 451 kB/s | 45 kB 00:00 2024-09-14T03:39:36.744 INFO:teuthology.orchestra.run.smithi083.stdout:(22/31): python3-ceph-common-19.3.0-4907.ga2782 2.8 MB/s | 143 kB 00:00 2024-09-14T03:39:36.770 INFO:teuthology.orchestra.run.smithi083.stdout:(23/31): python3-cephfs-19.3.0-4907.ga2782315.e 3.2 MB/s | 161 kB 00:00 2024-09-14T03:39:36.803 INFO:teuthology.orchestra.run.smithi083.stdout:(24/31): python3-rados-19.3.0-4907.ga2782315.el 5.3 MB/s | 320 kB 00:00 2024-09-14T03:39:36.829 INFO:teuthology.orchestra.run.smithi083.stdout:(25/31): python3-rbd-19.3.0-4907.ga2782315.el9. 5.1 MB/s | 305 kB 00:00 2024-09-14T03:39:36.854 INFO:teuthology.orchestra.run.smithi083.stdout:(26/31): python3-rgw-19.3.0-4907.ga2782315.el9. 1.9 MB/s | 99 kB 00:00 2024-09-14T03:39:36.896 INFO:teuthology.orchestra.run.smithi083.stdout:(27/31): rbd-fuse-19.3.0-4907.ga2782315.el9.x86 1.2 MB/s | 85 kB 00:00 2024-09-14T03:39:36.955 INFO:teuthology.orchestra.run.smithi083.stdout:(28/31): rbd-nbd-19.3.0-4907.ga2782315.el9.x86_ 2.9 MB/s | 171 kB 00:00 2024-09-14T03:39:37.022 INFO:teuthology.orchestra.run.smithi083.stdout:(29/31): rbd-mirror-19.3.0-4907.ga2782315.el9.x 19 MB/s | 3.2 MB 00:00 2024-09-14T03:39:37.055 INFO:teuthology.orchestra.run.smithi083.stdout:(30/31): ceph-mgr-modules-core-19.3.0-4907.ga27 2.8 MB/s | 291 kB 00:00 2024-09-14T03:39:37.614 INFO:teuthology.orchestra.run.smithi083.stdout:(31/31): ceph-test-19.3.0-4907.ga2782315.el9.x8 25 MB/s | 50 MB 00:02 2024-09-14T03:39:37.618 INFO:teuthology.orchestra.run.smithi083.stdout:-------------------------------------------------------------------------------- 2024-09-14T03:39:37.618 INFO:teuthology.orchestra.run.smithi083.stdout:Total 36 MB/s | 134 MB 00:03 2024-09-14T03:39:37.633 INFO:teuthology.orchestra.run.smithi083.stdout:Running transaction check 2024-09-14T03:39:37.690 INFO:teuthology.orchestra.run.smithi083.stdout:Transaction check succeeded. 2024-09-14T03:39:37.690 INFO:teuthology.orchestra.run.smithi083.stdout:Running transaction test 2024-09-14T03:39:38.585 INFO:teuthology.orchestra.run.smithi083.stdout:Transaction test succeeded. 2024-09-14T03:39:38.585 INFO:teuthology.orchestra.run.smithi083.stdout:Running transaction 2024-09-14T03:39:41.218 INFO:teuthology.orchestra.run.smithi083.stdout: Preparing : 1/1 2024-09-14T03:39:41.323 INFO:teuthology.orchestra.run.smithi083.stdout: Upgrading : librados2-2:19.3.0-4907.ga2782315.el9.x86_64 1/61 2024-09-14T03:39:41.429 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: librados2-2:19.3.0-4907.ga2782315.el9.x86_64 1/61 2024-09-14T03:39:41.474 INFO:teuthology.orchestra.run.smithi083.stdout: Upgrading : libcephfs2-2:19.3.0-4907.ga2782315.el9.x86_64 2/61 2024-09-14T03:39:42.078 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: libcephfs2-2:19.3.0-4907.ga2782315.el9.x86_64 2/61 2024-09-14T03:39:42.159 INFO:teuthology.orchestra.run.smithi083.stdout: Upgrading : librgw2-2:19.3.0-4907.ga2782315.el9.x86_64 3/61 2024-09-14T03:39:42.251 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: librgw2-2:19.3.0-4907.ga2782315.el9.x86_64 3/61 2024-09-14T03:39:42.295 INFO:teuthology.orchestra.run.smithi083.stdout: Upgrading : python3-rados-2:19.3.0-4907.ga2782315.el9.x86_64 4/61 2024-09-14T03:39:42.699 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : libnbd-1.20.2-2.el9.x86_64 5/61 2024-09-14T03:39:42.770 INFO:teuthology.orchestra.run.smithi083.stdout: Upgrading : librbd1-2:19.3.0-4907.ga2782315.el9.x86_64 6/61 2024-09-14T03:39:42.845 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: librbd1-2:19.3.0-4907.ga2782315.el9.x86_64 6/61 2024-09-14T03:39:42.880 INFO:teuthology.orchestra.run.smithi083.stdout: Upgrading : libcephsqlite-2:19.3.0-4907.ga2782315.el9.x86_64 7/61 2024-09-14T03:39:42.989 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: libcephsqlite-2:19.3.0-4907.ga2782315.el9.x86_64 7/61 2024-09-14T03:39:43.023 INFO:teuthology.orchestra.run.smithi083.stdout: Upgrading : libradosstriper1-2:19.3.0-4907.ga2782315.el9.x86_6 8/61 2024-09-14T03:39:43.090 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: libradosstriper1-2:19.3.0-4907.ga2782315.el9.x86_6 8/61 2024-09-14T03:39:43.140 INFO:teuthology.orchestra.run.smithi083.stdout: Upgrading : python3-ceph-argparse-2:19.3.0-4907.ga2782315.el9. 9/61 2024-09-14T03:39:43.213 INFO:teuthology.orchestra.run.smithi083.stdout: Upgrading : python3-cephfs-2:19.3.0-4907.ga2782315.el9.x86_64 10/61 2024-09-14T03:39:43.270 INFO:teuthology.orchestra.run.smithi083.stdout: Upgrading : python3-rbd-2:19.3.0-4907.ga2782315.el9.x86_64 11/61 2024-09-14T03:39:43.322 INFO:teuthology.orchestra.run.smithi083.stdout: Upgrading : python3-rgw-2:19.3.0-4907.ga2782315.el9.x86_64 12/61 2024-09-14T03:39:43.490 INFO:teuthology.orchestra.run.smithi083.stdout: Upgrading : librados-devel-2:19.3.0-4907.ga2782315.el9.x86_64 13/61 2024-09-14T03:39:43.602 INFO:teuthology.orchestra.run.smithi083.stdout: Upgrading : ceph-mgr-modules-core-2:19.3.0-4907.ga2782315.el9. 14/61 2024-09-14T03:39:43.663 INFO:teuthology.orchestra.run.smithi083.stdout: Upgrading : python3-ceph-common-2:19.3.0-4907.ga2782315.el9.x8 15/61 2024-09-14T03:39:46.189 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: ceph-common-2:19.3.0-4907.ga2782315.el9.x86_64 16/61 2024-09-14T03:39:46.256 INFO:teuthology.orchestra.run.smithi083.stdout: Upgrading : ceph-common-2:19.3.0-4907.ga2782315.el9.x86_64 16/61 2024-09-14T03:39:46.933 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: ceph-common-2:19.3.0-4907.ga2782315.el9.x86_64 16/61 2024-09-14T03:39:47.145 INFO:teuthology.orchestra.run.smithi083.stdout: Upgrading : ceph-base-2:19.3.0-4907.ga2782315.el9.x86_64 17/61 2024-09-14T03:39:47.233 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: ceph-base-2:19.3.0-4907.ga2782315.el9.x86_64 17/61 2024-09-14T03:39:47.273 INFO:teuthology.orchestra.run.smithi083.stdout: Upgrading : ceph-selinux-2:19.3.0-4907.ga2782315.el9.x86_64 18/61 2024-09-14T03:39:55.800 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: ceph-selinux-2:19.3.0-4907.ga2782315.el9.x86_64 18/61 2024-09-14T03:39:55.926 INFO:teuthology.orchestra.run.smithi083.stdout: Upgrading : ceph-mds-2:19.3.0-4907.ga2782315.el9.x86_64 19/61 2024-09-14T03:39:56.071 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: ceph-mds-2:19.3.0-4907.ga2782315.el9.x86_64 19/61 2024-09-14T03:39:56.144 INFO:teuthology.orchestra.run.smithi083.stdout: Upgrading : ceph-mgr-2:19.3.0-4907.ga2782315.el9.x86_64 20/61 2024-09-14T03:39:56.685 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: ceph-mgr-2:19.3.0-4907.ga2782315.el9.x86_64 20/61 2024-09-14T03:39:56.745 INFO:teuthology.orchestra.run.smithi083.stdout: Upgrading : ceph-mon-2:19.3.0-4907.ga2782315.el9.x86_64 21/61 2024-09-14T03:39:58.519 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: ceph-mon-2:19.3.0-4907.ga2782315.el9.x86_64 21/61 2024-09-14T03:39:58.648 INFO:teuthology.orchestra.run.smithi083.stdout: Upgrading : ceph-osd-2:19.3.0-4907.ga2782315.el9.x86_64 22/61 2024-09-14T03:39:58.660 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: ceph-osd-2:19.3.0-4907.ga2782315.el9.x86_64 22/61 2024-09-14T03:39:59.880 INFO:teuthology.orchestra.run.smithi083.stdout: Upgrading : ceph-2:19.3.0-4907.ga2782315.el9.x86_64 23/61 2024-09-14T03:39:59.933 INFO:teuthology.orchestra.run.smithi083.stdout: Upgrading : ceph-radosgw-2:19.3.0-4907.ga2782315.el9.x86_64 24/61 2024-09-14T03:39:59.955 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: ceph-radosgw-2:19.3.0-4907.ga2782315.el9.x86_64 24/61 2024-09-14T03:40:00.000 INFO:teuthology.orchestra.run.smithi083.stdout: Upgrading : ceph-immutable-object-cache-2:19.3.0-4907.ga278231 25/61 2024-09-14T03:40:00.369 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: ceph-immutable-object-cache-2:19.3.0-4907.ga278231 25/61 2024-09-14T03:40:00.409 INFO:teuthology.orchestra.run.smithi083.stdout: Upgrading : rbd-mirror-2:19.3.0-4907.ga2782315.el9.x86_64 26/61 2024-09-14T03:40:06.545 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: rbd-mirror-2:19.3.0-4907.ga2782315.el9.x86_64 26/61 2024-09-14T03:40:06.857 INFO:teuthology.orchestra.run.smithi083.stdout: Upgrading : ceph-test-2:19.3.0-4907.ga2782315.el9.x86_64 27/61 2024-09-14T03:40:06.902 INFO:teuthology.orchestra.run.smithi083.stdout: Upgrading : libcephfs-devel-2:19.3.0-4907.ga2782315.el9.x86_64 28/61 2024-09-14T03:40:06.952 INFO:teuthology.orchestra.run.smithi083.stdout: Upgrading : rbd-fuse-2:19.3.0-4907.ga2782315.el9.x86_64 29/61 2024-09-14T03:40:07.068 INFO:teuthology.orchestra.run.smithi083.stdout: Upgrading : rbd-nbd-2:19.3.0-4907.ga2782315.el9.x86_64 30/61 2024-09-14T03:40:07.155 INFO:teuthology.orchestra.run.smithi083.stdout: Upgrading : ceph-fuse-2:19.3.0-4907.ga2782315.el9.x86_64 31/61 2024-09-14T03:40:07.186 INFO:teuthology.orchestra.run.smithi083.stdout: Cleanup : ceph-2:19.1.1-324.gecab7be5.el9.x86_64 32/61 2024-09-14T03:40:07.225 INFO:teuthology.orchestra.run.smithi083.stdout: Cleanup : ceph-test-2:19.1.1-324.gecab7be5.el9.x86_64 33/61 2024-09-14T03:40:07.231 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: ceph-radosgw-2:19.1.1-324.gecab7be5.el9.x86_64 34/61 2024-09-14T03:40:07.231 INFO:teuthology.orchestra.run.smithi083.stdout: Cleanup : ceph-radosgw-2:19.1.1-324.gecab7be5.el9.x86_64 34/61 2024-09-14T03:40:07.276 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: ceph-radosgw-2:19.1.1-324.gecab7be5.el9.x86_64 34/61 2024-09-14T03:40:07.301 INFO:teuthology.orchestra.run.smithi083.stdout: Cleanup : rbd-nbd-2:19.1.1-324.gecab7be5.el9.x86_64 35/61 2024-09-14T03:40:07.306 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: rbd-mirror-2:19.1.1-324.gecab7be5.el9.x86_64 36/61 2024-09-14T03:40:07.306 INFO:teuthology.orchestra.run.smithi083.stdout: Cleanup : rbd-mirror-2:19.1.1-324.gecab7be5.el9.x86_64 36/61 2024-09-14T03:40:07.335 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: rbd-mirror-2:19.1.1-324.gecab7be5.el9.x86_64 36/61 2024-09-14T03:40:07.340 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: ceph-mgr-2:19.1.1-324.gecab7be5.el9.x86_64 37/61 2024-09-14T03:40:07.340 INFO:teuthology.orchestra.run.smithi083.stdout: Cleanup : ceph-mgr-2:19.1.1-324.gecab7be5.el9.x86_64 37/61 2024-09-14T03:40:07.369 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: ceph-mgr-2:19.1.1-324.gecab7be5.el9.x86_64 37/61 2024-09-14T03:40:07.394 INFO:teuthology.orchestra.run.smithi083.stdout: Cleanup : rbd-fuse-2:19.1.1-324.gecab7be5.el9.x86_64 38/61 2024-09-14T03:40:07.399 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: ceph-immutable-object-cache-2:19.1.1-324.gecab7be5 39/61 2024-09-14T03:40:07.399 INFO:teuthology.orchestra.run.smithi083.stdout: Cleanup : ceph-immutable-object-cache-2:19.1.1-324.gecab7be5 39/61 2024-09-14T03:40:07.428 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: ceph-immutable-object-cache-2:19.1.1-324.gecab7be5 39/61 2024-09-14T03:40:07.453 INFO:teuthology.orchestra.run.smithi083.stdout: Cleanup : libcephfs-devel-2:19.1.1-324.gecab7be5.el9.x86_64 40/61 2024-09-14T03:40:07.453 INFO:teuthology.orchestra.run.smithi083.stdout: Cleanup : libcephsqlite-2:19.1.1-324.gecab7be5.el9.x86_64 41/61 2024-09-14T03:40:07.529 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: libcephsqlite-2:19.1.1-324.gecab7be5.el9.x86_64 41/61 2024-09-14T03:40:07.554 INFO:teuthology.orchestra.run.smithi083.stdout: Cleanup : librados-devel-2:19.1.1-324.gecab7be5.el9.x86_64 42/61 2024-09-14T03:40:07.559 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: ceph-mds-2:19.1.1-324.gecab7be5.el9.x86_64 43/61 2024-09-14T03:40:07.559 INFO:teuthology.orchestra.run.smithi083.stdout: Cleanup : ceph-mds-2:19.1.1-324.gecab7be5.el9.x86_64 43/61 2024-09-14T03:40:07.588 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: ceph-mds-2:19.1.1-324.gecab7be5.el9.x86_64 43/61 2024-09-14T03:40:07.593 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: ceph-mon-2:19.1.1-324.gecab7be5.el9.x86_64 44/61 2024-09-14T03:40:07.593 INFO:teuthology.orchestra.run.smithi083.stdout: Cleanup : ceph-mon-2:19.1.1-324.gecab7be5.el9.x86_64 44/61 2024-09-14T03:40:07.622 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: ceph-mon-2:19.1.1-324.gecab7be5.el9.x86_64 44/61 2024-09-14T03:40:07.627 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: ceph-osd-2:19.1.1-324.gecab7be5.el9.x86_64 45/61 2024-09-14T03:40:07.627 INFO:teuthology.orchestra.run.smithi083.stdout: Cleanup : ceph-osd-2:19.1.1-324.gecab7be5.el9.x86_64 45/61 2024-09-14T03:40:07.664 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: ceph-osd-2:19.1.1-324.gecab7be5.el9.x86_64 45/61 2024-09-14T03:40:07.664 INFO:teuthology.orchestra.run.smithi083.stdout: Cleanup : ceph-selinux-2:19.1.1-324.gecab7be5.el9.x86_64 46/61 2024-09-14T03:40:07.700 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: ceph-selinux-2:19.1.1-324.gecab7be5.el9.x86_64 46/61 2024-09-14T03:40:07.704 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: ceph-base-2:19.1.1-324.gecab7be5.el9.x86_64 47/61 2024-09-14T03:40:07.704 INFO:teuthology.orchestra.run.smithi083.stdout: Cleanup : ceph-base-2:19.1.1-324.gecab7be5.el9.x86_64 47/61 2024-09-14T03:40:07.809 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: ceph-base-2:19.1.1-324.gecab7be5.el9.x86_64 47/61 2024-09-14T03:40:07.810 INFO:teuthology.orchestra.run.smithi083.stdout: Cleanup : ceph-common-2:19.1.1-324.gecab7be5.el9.x86_64 48/61 2024-09-14T03:40:07.843 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: ceph-common-2:19.1.1-324.gecab7be5.el9.x86_64 48/61 2024-09-14T03:40:07.843 INFO:teuthology.orchestra.run.smithi083.stdout: Cleanup : libradosstriper1-2:19.1.1-324.gecab7be5.el9.x86_64 49/61 2024-09-14T03:40:07.927 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: libradosstriper1-2:19.1.1-324.gecab7be5.el9.x86_64 49/61 2024-09-14T03:40:07.952 INFO:teuthology.orchestra.run.smithi083.stdout: Cleanup : python3-cephfs-2:19.1.1-324.gecab7be5.el9.x86_64 50/61 2024-09-14T03:40:07.978 INFO:teuthology.orchestra.run.smithi083.stdout: Cleanup : python3-rbd-2:19.1.1-324.gecab7be5.el9.x86_64 51/61 2024-09-14T03:40:07.978 INFO:teuthology.orchestra.run.smithi083.stdout: Cleanup : librbd1-2:19.1.1-324.gecab7be5.el9.x86_64 52/61 2024-09-14T03:40:08.063 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: librbd1-2:19.1.1-324.gecab7be5.el9.x86_64 52/61 2024-09-14T03:40:08.088 INFO:teuthology.orchestra.run.smithi083.stdout: Cleanup : python3-rgw-2:19.1.1-324.gecab7be5.el9.x86_64 53/61 2024-09-14T03:40:08.088 INFO:teuthology.orchestra.run.smithi083.stdout: Cleanup : librgw2-2:19.1.1-324.gecab7be5.el9.x86_64 54/61 2024-09-14T03:40:08.172 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: librgw2-2:19.1.1-324.gecab7be5.el9.x86_64 54/61 2024-09-14T03:40:08.197 INFO:teuthology.orchestra.run.smithi083.stdout: Cleanup : python3-rados-2:19.1.1-324.gecab7be5.el9.x86_64 55/61 2024-09-14T03:40:08.198 INFO:teuthology.orchestra.run.smithi083.stdout: Cleanup : libcephfs2-2:19.1.1-324.gecab7be5.el9.x86_64 56/61 2024-09-14T03:40:08.282 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: libcephfs2-2:19.1.1-324.gecab7be5.el9.x86_64 56/61 2024-09-14T03:40:08.307 INFO:teuthology.orchestra.run.smithi083.stdout: Cleanup : ceph-fuse-2:19.1.1-324.gecab7be5.el9.x86_64 57/61 2024-09-14T03:40:08.332 INFO:teuthology.orchestra.run.smithi083.stdout: Cleanup : python3-ceph-argparse-2:19.1.1-324.gecab7be5.el9.x 58/61 2024-09-14T03:40:08.357 INFO:teuthology.orchestra.run.smithi083.stdout: Cleanup : python3-ceph-common-2:19.1.1-324.gecab7be5.el9.x86 59/61 2024-09-14T03:40:08.383 INFO:teuthology.orchestra.run.smithi083.stdout: Cleanup : ceph-mgr-modules-core-2:19.1.1-324.gecab7be5.el9.n 60/61 2024-09-14T03:40:08.383 INFO:teuthology.orchestra.run.smithi083.stdout: Cleanup : librados2-2:19.1.1-324.gecab7be5.el9.x86_64 61/61 2024-09-14T03:40:09.589 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: librados2-2:19.1.1-324.gecab7be5.el9.x86_64 61/61 2024-09-14T03:40:09.589 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : libnbd-1.20.2-2.el9.x86_64 1/61 2024-09-14T03:40:09.589 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : ceph-2:19.3.0-4907.ga2782315.el9.x86_64 2/61 2024-09-14T03:40:09.589 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : ceph-2:19.1.1-324.gecab7be5.el9.x86_64 3/61 2024-09-14T03:40:09.589 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : ceph-base-2:19.3.0-4907.ga2782315.el9.x86_64 4/61 2024-09-14T03:40:09.589 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : ceph-base-2:19.1.1-324.gecab7be5.el9.x86_64 5/61 2024-09-14T03:40:09.590 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : ceph-common-2:19.3.0-4907.ga2782315.el9.x86_64 6/61 2024-09-14T03:40:09.590 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : ceph-common-2:19.1.1-324.gecab7be5.el9.x86_64 7/61 2024-09-14T03:40:09.590 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : ceph-fuse-2:19.3.0-4907.ga2782315.el9.x86_64 8/61 2024-09-14T03:40:09.590 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : ceph-fuse-2:19.1.1-324.gecab7be5.el9.x86_64 9/61 2024-09-14T03:40:09.590 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : ceph-immutable-object-cache-2:19.3.0-4907.ga278231 10/61 2024-09-14T03:40:09.590 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : ceph-immutable-object-cache-2:19.1.1-324.gecab7be5 11/61 2024-09-14T03:40:09.590 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : ceph-mds-2:19.3.0-4907.ga2782315.el9.x86_64 12/61 2024-09-14T03:40:09.590 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : ceph-mds-2:19.1.1-324.gecab7be5.el9.x86_64 13/61 2024-09-14T03:40:09.590 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : ceph-mgr-2:19.3.0-4907.ga2782315.el9.x86_64 14/61 2024-09-14T03:40:09.590 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : ceph-mgr-2:19.1.1-324.gecab7be5.el9.x86_64 15/61 2024-09-14T03:40:09.590 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : ceph-mon-2:19.3.0-4907.ga2782315.el9.x86_64 16/61 2024-09-14T03:40:09.590 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : ceph-mon-2:19.1.1-324.gecab7be5.el9.x86_64 17/61 2024-09-14T03:40:09.590 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : ceph-osd-2:19.3.0-4907.ga2782315.el9.x86_64 18/61 2024-09-14T03:40:09.590 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : ceph-osd-2:19.1.1-324.gecab7be5.el9.x86_64 19/61 2024-09-14T03:40:09.590 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : ceph-radosgw-2:19.3.0-4907.ga2782315.el9.x86_64 20/61 2024-09-14T03:40:09.591 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : ceph-radosgw-2:19.1.1-324.gecab7be5.el9.x86_64 21/61 2024-09-14T03:40:09.591 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : ceph-selinux-2:19.3.0-4907.ga2782315.el9.x86_64 22/61 2024-09-14T03:40:09.591 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : ceph-selinux-2:19.1.1-324.gecab7be5.el9.x86_64 23/61 2024-09-14T03:40:09.591 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : ceph-test-2:19.3.0-4907.ga2782315.el9.x86_64 24/61 2024-09-14T03:40:09.591 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : ceph-test-2:19.1.1-324.gecab7be5.el9.x86_64 25/61 2024-09-14T03:40:09.591 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : libcephfs-devel-2:19.3.0-4907.ga2782315.el9.x86_64 26/61 2024-09-14T03:40:09.591 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : libcephfs-devel-2:19.1.1-324.gecab7be5.el9.x86_64 27/61 2024-09-14T03:40:09.594 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : libcephfs2-2:19.3.0-4907.ga2782315.el9.x86_64 28/61 2024-09-14T03:40:09.594 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : libcephfs2-2:19.1.1-324.gecab7be5.el9.x86_64 29/61 2024-09-14T03:40:09.594 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : libcephsqlite-2:19.3.0-4907.ga2782315.el9.x86_64 30/61 2024-09-14T03:40:09.594 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : libcephsqlite-2:19.1.1-324.gecab7be5.el9.x86_64 31/61 2024-09-14T03:40:09.595 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : librados-devel-2:19.3.0-4907.ga2782315.el9.x86_64 32/61 2024-09-14T03:40:09.595 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : librados-devel-2:19.1.1-324.gecab7be5.el9.x86_64 33/61 2024-09-14T03:40:09.595 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : librados2-2:19.3.0-4907.ga2782315.el9.x86_64 34/61 2024-09-14T03:40:09.595 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : librados2-2:19.1.1-324.gecab7be5.el9.x86_64 35/61 2024-09-14T03:40:09.595 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : libradosstriper1-2:19.3.0-4907.ga2782315.el9.x86_6 36/61 2024-09-14T03:40:09.595 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : libradosstriper1-2:19.1.1-324.gecab7be5.el9.x86_64 37/61 2024-09-14T03:40:09.595 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : librbd1-2:19.3.0-4907.ga2782315.el9.x86_64 38/61 2024-09-14T03:40:09.595 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : librbd1-2:19.1.1-324.gecab7be5.el9.x86_64 39/61 2024-09-14T03:40:09.595 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : librgw2-2:19.3.0-4907.ga2782315.el9.x86_64 40/61 2024-09-14T03:40:09.595 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : librgw2-2:19.1.1-324.gecab7be5.el9.x86_64 41/61 2024-09-14T03:40:09.595 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : python3-ceph-argparse-2:19.3.0-4907.ga2782315.el9. 42/61 2024-09-14T03:40:09.595 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : python3-ceph-argparse-2:19.1.1-324.gecab7be5.el9.x 43/61 2024-09-14T03:40:09.595 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : python3-ceph-common-2:19.3.0-4907.ga2782315.el9.x8 44/61 2024-09-14T03:40:09.595 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : python3-ceph-common-2:19.1.1-324.gecab7be5.el9.x86 45/61 2024-09-14T03:40:09.596 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : python3-cephfs-2:19.3.0-4907.ga2782315.el9.x86_64 46/61 2024-09-14T03:40:09.596 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : python3-cephfs-2:19.1.1-324.gecab7be5.el9.x86_64 47/61 2024-09-14T03:40:09.596 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : python3-rados-2:19.3.0-4907.ga2782315.el9.x86_64 48/61 2024-09-14T03:40:09.596 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : python3-rados-2:19.1.1-324.gecab7be5.el9.x86_64 49/61 2024-09-14T03:40:09.596 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : python3-rbd-2:19.3.0-4907.ga2782315.el9.x86_64 50/61 2024-09-14T03:40:09.596 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : python3-rbd-2:19.1.1-324.gecab7be5.el9.x86_64 51/61 2024-09-14T03:40:09.596 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : python3-rgw-2:19.3.0-4907.ga2782315.el9.x86_64 52/61 2024-09-14T03:40:09.596 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : python3-rgw-2:19.1.1-324.gecab7be5.el9.x86_64 53/61 2024-09-14T03:40:09.596 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : rbd-fuse-2:19.3.0-4907.ga2782315.el9.x86_64 54/61 2024-09-14T03:40:09.596 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : rbd-fuse-2:19.1.1-324.gecab7be5.el9.x86_64 55/61 2024-09-14T03:40:09.596 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : rbd-mirror-2:19.3.0-4907.ga2782315.el9.x86_64 56/61 2024-09-14T03:40:09.596 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : rbd-mirror-2:19.1.1-324.gecab7be5.el9.x86_64 57/61 2024-09-14T03:40:09.596 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : rbd-nbd-2:19.3.0-4907.ga2782315.el9.x86_64 58/61 2024-09-14T03:40:09.596 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : rbd-nbd-2:19.1.1-324.gecab7be5.el9.x86_64 59/61 2024-09-14T03:40:09.596 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : ceph-mgr-modules-core-2:19.3.0-4907.ga2782315.el9. 60/61 2024-09-14T03:40:10.710 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : ceph-mgr-modules-core-2:19.1.1-324.gecab7be5.el9.n 61/61 2024-09-14T03:40:10.710 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:40:10.710 INFO:teuthology.orchestra.run.smithi083.stdout:Upgraded: 2024-09-14T03:40:10.710 INFO:teuthology.orchestra.run.smithi083.stdout: ceph-2:19.3.0-4907.ga2782315.el9.x86_64 2024-09-14T03:40:10.710 INFO:teuthology.orchestra.run.smithi083.stdout: ceph-base-2:19.3.0-4907.ga2782315.el9.x86_64 2024-09-14T03:40:10.710 INFO:teuthology.orchestra.run.smithi083.stdout: ceph-common-2:19.3.0-4907.ga2782315.el9.x86_64 2024-09-14T03:40:10.710 INFO:teuthology.orchestra.run.smithi083.stdout: ceph-fuse-2:19.3.0-4907.ga2782315.el9.x86_64 2024-09-14T03:40:10.710 INFO:teuthology.orchestra.run.smithi083.stdout: ceph-immutable-object-cache-2:19.3.0-4907.ga2782315.el9.x86_64 2024-09-14T03:40:10.710 INFO:teuthology.orchestra.run.smithi083.stdout: ceph-mds-2:19.3.0-4907.ga2782315.el9.x86_64 2024-09-14T03:40:10.710 INFO:teuthology.orchestra.run.smithi083.stdout: ceph-mgr-2:19.3.0-4907.ga2782315.el9.x86_64 2024-09-14T03:40:10.711 INFO:teuthology.orchestra.run.smithi083.stdout: ceph-mgr-modules-core-2:19.3.0-4907.ga2782315.el9.noarch 2024-09-14T03:40:10.711 INFO:teuthology.orchestra.run.smithi083.stdout: ceph-mon-2:19.3.0-4907.ga2782315.el9.x86_64 2024-09-14T03:40:10.711 INFO:teuthology.orchestra.run.smithi083.stdout: ceph-osd-2:19.3.0-4907.ga2782315.el9.x86_64 2024-09-14T03:40:10.711 INFO:teuthology.orchestra.run.smithi083.stdout: ceph-radosgw-2:19.3.0-4907.ga2782315.el9.x86_64 2024-09-14T03:40:10.711 INFO:teuthology.orchestra.run.smithi083.stdout: ceph-selinux-2:19.3.0-4907.ga2782315.el9.x86_64 2024-09-14T03:40:10.711 INFO:teuthology.orchestra.run.smithi083.stdout: ceph-test-2:19.3.0-4907.ga2782315.el9.x86_64 2024-09-14T03:40:10.711 INFO:teuthology.orchestra.run.smithi083.stdout: libcephfs-devel-2:19.3.0-4907.ga2782315.el9.x86_64 2024-09-14T03:40:10.711 INFO:teuthology.orchestra.run.smithi083.stdout: libcephfs2-2:19.3.0-4907.ga2782315.el9.x86_64 2024-09-14T03:40:10.711 INFO:teuthology.orchestra.run.smithi083.stdout: libcephsqlite-2:19.3.0-4907.ga2782315.el9.x86_64 2024-09-14T03:40:10.711 INFO:teuthology.orchestra.run.smithi083.stdout: librados-devel-2:19.3.0-4907.ga2782315.el9.x86_64 2024-09-14T03:40:10.711 INFO:teuthology.orchestra.run.smithi083.stdout: librados2-2:19.3.0-4907.ga2782315.el9.x86_64 2024-09-14T03:40:10.711 INFO:teuthology.orchestra.run.smithi083.stdout: libradosstriper1-2:19.3.0-4907.ga2782315.el9.x86_64 2024-09-14T03:40:10.712 INFO:teuthology.orchestra.run.smithi083.stdout: librbd1-2:19.3.0-4907.ga2782315.el9.x86_64 2024-09-14T03:40:10.712 INFO:teuthology.orchestra.run.smithi083.stdout: librgw2-2:19.3.0-4907.ga2782315.el9.x86_64 2024-09-14T03:40:10.712 INFO:teuthology.orchestra.run.smithi083.stdout: python3-ceph-argparse-2:19.3.0-4907.ga2782315.el9.x86_64 2024-09-14T03:40:10.712 INFO:teuthology.orchestra.run.smithi083.stdout: python3-ceph-common-2:19.3.0-4907.ga2782315.el9.x86_64 2024-09-14T03:40:10.712 INFO:teuthology.orchestra.run.smithi083.stdout: python3-cephfs-2:19.3.0-4907.ga2782315.el9.x86_64 2024-09-14T03:40:10.712 INFO:teuthology.orchestra.run.smithi083.stdout: python3-rados-2:19.3.0-4907.ga2782315.el9.x86_64 2024-09-14T03:40:10.713 INFO:teuthology.orchestra.run.smithi083.stdout: python3-rbd-2:19.3.0-4907.ga2782315.el9.x86_64 2024-09-14T03:40:10.713 INFO:teuthology.orchestra.run.smithi083.stdout: python3-rgw-2:19.3.0-4907.ga2782315.el9.x86_64 2024-09-14T03:40:10.713 INFO:teuthology.orchestra.run.smithi083.stdout: rbd-fuse-2:19.3.0-4907.ga2782315.el9.x86_64 2024-09-14T03:40:10.713 INFO:teuthology.orchestra.run.smithi083.stdout: rbd-mirror-2:19.3.0-4907.ga2782315.el9.x86_64 2024-09-14T03:40:10.713 INFO:teuthology.orchestra.run.smithi083.stdout: rbd-nbd-2:19.3.0-4907.ga2782315.el9.x86_64 2024-09-14T03:40:10.713 INFO:teuthology.orchestra.run.smithi083.stdout:Installed: 2024-09-14T03:40:10.713 INFO:teuthology.orchestra.run.smithi083.stdout: libnbd-1.20.2-2.el9.x86_64 2024-09-14T03:40:10.713 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:40:10.714 INFO:teuthology.orchestra.run.smithi083.stdout:Complete! 2024-09-14T03:40:11.113 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&sha1=a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T03:40:11.255 DEBUG:teuthology.orchestra.run.smithi083:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2024-09-14T03:40:11.277 INFO:teuthology.orchestra.run.smithi083.stdout:19.3.0-4907.ga2782315.el9 2024-09-14T03:40:11.278 INFO:teuthology.packaging:The installed version of ceph is 19.3.0-4907.ga2782315.el9 2024-09-14T03:40:11.278 INFO:teuthology.task.install:The correct ceph version 19.3.0-4907.ga2782315 is installed. 2024-09-14T03:40:11.279 INFO:teuthology.run_tasks:Running task print... 2024-09-14T03:40:11.288 INFO:teuthology.task.print:**** done install.upgrade 2024-09-14T03:40:11.289 INFO:teuthology.run_tasks:Running task ceph.restart... 2024-09-14T03:40:11.315 DEBUG:tasks.ceph.mon.a:waiting for process to exit 2024-09-14T03:40:11.315 INFO:teuthology.orchestra.run:waiting for 300 2024-09-14T03:40:11.329 INFO:tasks.ceph.mon.a:Stopped 2024-09-14T03:40:11.330 INFO:tasks.ceph.mon.a:Restarting daemon 2024-09-14T03:40:11.330 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mon -f --cluster ceph -i a 2024-09-14T03:40:11.372 INFO:tasks.ceph.mon.a:Started 2024-09-14T03:40:11.372 DEBUG:tasks.ceph.mon.b:waiting for process to exit 2024-09-14T03:40:11.372 INFO:teuthology.orchestra.run:waiting for 300 2024-09-14T03:40:11.384 INFO:tasks.ceph.mon.b:Stopped 2024-09-14T03:40:11.385 INFO:tasks.ceph.mon.b:Restarting daemon 2024-09-14T03:40:11.385 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mon -f --cluster ceph -i b 2024-09-14T03:40:11.426 INFO:tasks.ceph.mon.b:Started 2024-09-14T03:40:11.427 DEBUG:tasks.ceph.mon.c:waiting for process to exit 2024-09-14T03:40:11.427 INFO:teuthology.orchestra.run:waiting for 300 2024-09-14T03:40:11.439 INFO:tasks.ceph.mon.c:Stopped 2024-09-14T03:40:11.439 INFO:tasks.ceph.mon.c:Restarting daemon 2024-09-14T03:40:11.439 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mon -f --cluster ceph -i c 2024-09-14T03:40:11.482 INFO:tasks.ceph.mon.c:Started 2024-09-14T03:40:11.482 DEBUG:tasks.ceph.mgr.x:waiting for process to exit 2024-09-14T03:40:11.482 INFO:teuthology.orchestra.run:waiting for 300 2024-09-14T03:40:11.502 INFO:tasks.ceph.mgr.x:Stopped 2024-09-14T03:40:11.502 INFO:tasks.ceph.mgr.x:Restarting daemon 2024-09-14T03:40:11.502 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mgr -f --cluster ceph -i x 2024-09-14T03:40:11.546 INFO:tasks.ceph.mgr.x:Started 2024-09-14T03:40:11.546 DEBUG:tasks.ceph.mgr.y:waiting for process to exit 2024-09-14T03:40:11.546 INFO:teuthology.orchestra.run:waiting for 300 2024-09-14T03:40:11.572 INFO:tasks.ceph.mgr.y:Stopped 2024-09-14T03:40:11.572 INFO:tasks.ceph.mgr.y:Restarting daemon 2024-09-14T03:40:11.572 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mgr -f --cluster ceph -i y 2024-09-14T03:40:11.614 INFO:tasks.ceph.mgr.y:Started 2024-09-14T03:40:11.615 INFO:teuthology.run_tasks:Running task ceph.healthy... 2024-09-14T03:40:11.624 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-09-14T03:40:11.624 INFO:tasks.ceph.ceph_manager.ceph:waiting for mgr available 2024-09-14T03:40:11.624 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph mgr dump --format=json 2024-09-14T03:40:17.729 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:17.727+0000 7efe52190640 1 -- 172.21.15.83:0/4102909132 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7efe4c0999a0 msgr2=0x7efe2c00bb10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:17.729 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:17.727+0000 7efe52190640 1 --2- 172.21.15.83:0/4102909132 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7efe4c0999a0 0x7efe2c00bb10 secure :-1 s=READY pgs=16 cs=0 l=1 rev1=1 crypto rx=0x7efe34005da0 tx=0x7efe3402bd20 comp rx=0 tx=0).stop 2024-09-14T03:40:17.729 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:17.727+0000 7efe52190640 1 -- 172.21.15.83:0/4102909132 shutdown_connections 2024-09-14T03:40:17.730 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:17.727+0000 7efe52190640 1 --2- 172.21.15.83:0/4102909132 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7efe2c003670 0x7efe2c005ae0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:17.730 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:17.727+0000 7efe52190640 1 --2- 172.21.15.83:0/4102909132 >> v2:172.21.15.83:3300/0 conn(0x7efe4c099080 0x7efe2c00e110 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:17.730 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:17.727+0000 7efe52190640 1 --2- 172.21.15.83:0/4102909132 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7efe4c0999a0 0x7efe2c00bb10 unknown :-1 s=CLOSED pgs=16 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:17.730 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:17.727+0000 7efe52190640 1 -- 172.21.15.83:0/4102909132 >> 172.21.15.83:0/4102909132 conn(0x7efe4c0a1800 msgr2=0x7efe4c0a1c10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:17.730 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:17.727+0000 7efe52190640 1 -- 172.21.15.83:0/4102909132 shutdown_connections 2024-09-14T03:40:17.730 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:17.727+0000 7efe52190640 1 -- 172.21.15.83:0/4102909132 wait complete. 2024-09-14T03:40:17.730 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:17.728+0000 7efe52190640 1 Processor -- start 2024-09-14T03:40:17.730 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:17.728+0000 7efe52190640 1 -- start start 2024-09-14T03:40:17.732 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:17.728+0000 7efe52190640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7efe2c003670 0x7efe4c132e60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:17.732 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:17.728+0000 7efe52190640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7efe4c099080 0x7efe4c1333a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:17.732 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:17.728+0000 7efe52190640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7efe4c0999a0 0x7efe4c1338e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:17.732 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:17.728+0000 7efe52190640 1 -- --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7efe4c12ee40 con 0x7efe2c003670 2024-09-14T03:40:17.733 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:17.728+0000 7efe52190640 1 -- --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7efe4c12ecc0 con 0x7efe4c099080 2024-09-14T03:40:17.733 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:17.728+0000 7efe52190640 1 -- --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7efe4c12f410 con 0x7efe4c0999a0 2024-09-14T03:40:17.733 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:17.728+0000 7efe4bfff640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7efe4c0999a0 0x7efe4c1338e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:17.733 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:17.728+0000 7efe4b7fe640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7efe2c003670 0x7efe4c132e60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:17.733 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:17.728+0000 7efe4b7fe640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7efe2c003670 0x7efe4c132e60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3300/0 says I am v2:172.21.15.83:48310/0 (socket says 172.21.15.83:48310) 2024-09-14T03:40:17.733 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:17.728+0000 7efe4bfff640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7efe4c0999a0 0x7efe4c1338e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3302/0 says I am v2:172.21.15.83:58300/0 (socket says 172.21.15.83:58300) 2024-09-14T03:40:17.733 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:17.728+0000 7efe4b7fe640 1 -- 172.21.15.83:0/2861665550 learned_addr learned my addr 172.21.15.83:0/2861665550 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:40:17.733 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:17.728+0000 7efe43fff640 1 --2- 172.21.15.83:0/2861665550 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7efe4c099080 0x7efe4c1333a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:17.733 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:17.729+0000 7efe4bfff640 1 -- 172.21.15.83:0/2861665550 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7efe4c099080 msgr2=0x7efe4c1333a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:17.733 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:17.729+0000 7efe4bfff640 1 --2- 172.21.15.83:0/2861665550 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7efe4c099080 0x7efe4c1333a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:17.733 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:17.729+0000 7efe4bfff640 1 -- 172.21.15.83:0/2861665550 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7efe2c003670 msgr2=0x7efe4c132e60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:17.733 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:17.729+0000 7efe4bfff640 1 --2- 172.21.15.83:0/2861665550 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7efe2c003670 0x7efe4c132e60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:17.733 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:17.729+0000 7efe4bfff640 1 -- 172.21.15.83:0/2861665550 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7efe4c134080 con 0x7efe4c0999a0 2024-09-14T03:40:17.734 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:17.729+0000 7efe4b7fe640 1 --2- 172.21.15.83:0/2861665550 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7efe2c003670 0x7efe4c132e60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:40:17.734 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:17.729+0000 7efe43fff640 1 --2- 172.21.15.83:0/2861665550 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7efe4c099080 0x7efe4c1333a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:40:17.734 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:17.729+0000 7efe4bfff640 1 --2- 172.21.15.83:0/2861665550 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7efe4c0999a0 0x7efe4c1338e0 secure :-1 s=READY pgs=17 cs=0 l=1 rev1=1 crypto rx=0x7efe440062c0 tx=0x7efe44006780 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:40:17.734 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:17.729+0000 7efe48ff9640 1 -- 172.21.15.83:0/2861665550 <== mon.2 v2:172.21.15.83:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7efe44015070 con 0x7efe4c0999a0 2024-09-14T03:40:17.734 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:17.729+0000 7efe48ff9640 1 -- 172.21.15.83:0/2861665550 <== mon.2 v2:172.21.15.83:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7efe4400fdd0 con 0x7efe4c0999a0 2024-09-14T03:40:17.734 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:17.729+0000 7efe52190640 1 -- 172.21.15.83:0/2861665550 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7efe4c1294d0 con 0x7efe4c0999a0 2024-09-14T03:40:17.734 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:17.729+0000 7efe48ff9640 1 -- 172.21.15.83:0/2861665550 <== mon.2 v2:172.21.15.83:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7efe44011670 con 0x7efe4c0999a0 2024-09-14T03:40:17.734 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:17.729+0000 7efe52190640 1 -- 172.21.15.83:0/2861665550 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7efe4c134e50 con 0x7efe4c0999a0 2024-09-14T03:40:17.734 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:17.730+0000 7efe52190640 1 -- 172.21.15.83:0/2861665550 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7efe4c12f970 con 0x7efe4c0999a0 2024-09-14T03:40:17.734 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:17.731+0000 7efe48ff9640 1 -- 172.21.15.83:0/2861665550 <== mon.2 v2:172.21.15.83:3302/0 4 ==== mgrmap(e 5) ==== 55999+0+0 (secure 0 0 0) 0x7efe44011810 con 0x7efe4c0999a0 2024-09-14T03:40:17.734 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:17.731+0000 7efe48ff9640 1 -- 172.21.15.83:0/2861665550 <== mon.2 v2:172.21.15.83:3302/0 5 ==== osd_map(20..20 src has 1..20) ==== 2812+0+0 (secure 0 0 0) 0x7efe4405e810 con 0x7efe4c0999a0 2024-09-14T03:40:17.734 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:17.731+0000 7efe4b7fe640 1 -- 172.21.15.83:0/2861665550 >> v1:172.21.15.83:6800/1934165865 conn(0x7efe2c048840 legacy=0x7efe2c04ad00 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v1:172.21.15.83:6800/1934165865 2024-09-14T03:40:17.736 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:17.735+0000 7efe48ff9640 1 -- 172.21.15.83:0/2861665550 <== mon.2 v2:172.21.15.83:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+152684 (secure 0 0 0) 0x7efe4c1294d0 con 0x7efe4c0999a0 2024-09-14T03:40:17.915 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:17.913+0000 7efe52190640 1 -- 172.21.15.83:0/2861665550 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_command({"prefix": "mgr dump", "format": "json"} v 0) -- 0x7efe4c12adf0 con 0x7efe4c0999a0 2024-09-14T03:40:17.917 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:17.915+0000 7efe48ff9640 1 -- 172.21.15.83:0/2861665550 <== mon.2 v2:172.21.15.83:3302/0 7 ==== mon_command_ack([{"prefix": "mgr dump", "format": "json"}]=0 v5) ==== 74+0+108900 (secure 0 0 0) 0x7efe4405d9a0 con 0x7efe4c0999a0 2024-09-14T03:40:17.918 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:40:17.920 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:17.917+0000 7efe52190640 1 -- 172.21.15.83:0/2861665550 >> v1:172.21.15.83:6800/1934165865 conn(0x7efe2c048840 legacy=0x7efe2c04ad00 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-09-14T03:40:17.920 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:17.917+0000 7efe52190640 1 -- 172.21.15.83:0/2861665550 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7efe4c0999a0 msgr2=0x7efe4c1338e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:17.920 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:17.917+0000 7efe52190640 1 --2- 172.21.15.83:0/2861665550 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7efe4c0999a0 0x7efe4c1338e0 secure :-1 s=READY pgs=17 cs=0 l=1 rev1=1 crypto rx=0x7efe440062c0 tx=0x7efe44006780 comp rx=0 tx=0).stop 2024-09-14T03:40:17.920 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:17.917+0000 7efe52190640 1 -- 172.21.15.83:0/2861665550 shutdown_connections 2024-09-14T03:40:17.920 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:17.917+0000 7efe52190640 1 --2- 172.21.15.83:0/2861665550 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7efe4c0999a0 0x7efe4c1338e0 unknown :-1 s=CLOSED pgs=17 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:17.920 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:17.917+0000 7efe52190640 1 --2- 172.21.15.83:0/2861665550 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7efe4c099080 0x7efe4c1333a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:17.920 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:17.917+0000 7efe52190640 1 --2- 172.21.15.83:0/2861665550 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7efe2c003670 0x7efe4c132e60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:17.920 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:17.917+0000 7efe52190640 1 -- 172.21.15.83:0/2861665550 >> 172.21.15.83:0/2861665550 conn(0x7efe4c0a1800 msgr2=0x7efe4c1372f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:17.920 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:17.917+0000 7efe52190640 1 -- 172.21.15.83:0/2861665550 shutdown_connections 2024-09-14T03:40:17.920 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:17.917+0000 7efe52190640 1 -- 172.21.15.83:0/2861665550 wait complete. 2024-09-14T03:40:17.931 INFO:teuthology.orchestra.run.smithi083.stdout:{"epoch":5,"flags":0,"active_gid":4146,"active_name":"y","active_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6800","nonce":1934165865}]},"active_addr":"172.21.15.83:6800/1934165865","active_change":"2024-09-14T03:38:46.888221+0000","active_mgr_features":4540701547738038271,"available":true,"standbys":[{"gid":4099,"name":"x","mgr_features":4540701547738038271,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"secondary_zone_period_retry_limit":{"name":"secondary_zone_period_retry_limit","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"RGW module period update retry limit for secondary site","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["iostat","nfs","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"secondary_zone_period_retry_limit":{"name":"secondary_zone_period_retry_limit","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"RGW module period update retry limit for secondary site","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"squid":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":0,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.83:0","nonce":4246762177}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.83:0","nonce":293392365}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.83:0","nonce":3623382676}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.83:0","nonce":4208089382}]}]} 2024-09-14T03:40:17.933 INFO:tasks.ceph.ceph_manager.ceph:mgr available! 2024-09-14T03:40:17.934 INFO:tasks.ceph.ceph_manager.ceph:waiting for all up 2024-09-14T03:40:17.934 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-09-14T03:40:18.037 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.034+0000 7ffa1f7bb640 1 -- 172.21.15.83:0/1979877749 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7ffa180999a0 msgr2=0x7ffa1812eab0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:18.037 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.034+0000 7ffa1f7bb640 1 --2- 172.21.15.83:0/1979877749 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7ffa180999a0 0x7ffa1812eab0 secure :-1 s=READY pgs=18 cs=0 l=1 rev1=1 crypto rx=0x7ffa08005fd0 tx=0x7ffa0802a4a0 comp rx=0 tx=0).stop 2024-09-14T03:40:18.038 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.034+0000 7ffa1f7bb640 1 -- 172.21.15.83:0/1979877749 shutdown_connections 2024-09-14T03:40:18.038 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.034+0000 7ffa1f7bb640 1 --2- 172.21.15.83:0/1979877749 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7ffa1812eff0 0x7ffa1812f440 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:18.038 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.034+0000 7ffa1f7bb640 1 --2- 172.21.15.83:0/1979877749 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7ffa180999a0 0x7ffa1812eab0 unknown :-1 s=CLOSED pgs=18 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:18.038 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.034+0000 7ffa1f7bb640 1 --2- 172.21.15.83:0/1979877749 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7ffa18099080 0x7ffa18099460 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:18.038 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.034+0000 7ffa1f7bb640 1 -- 172.21.15.83:0/1979877749 >> 172.21.15.83:0/1979877749 conn(0x7ffa180a1800 msgr2=0x7ffa180a1c10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:18.038 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.035+0000 7ffa1f7bb640 1 -- 172.21.15.83:0/1979877749 shutdown_connections 2024-09-14T03:40:18.038 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.035+0000 7ffa1f7bb640 1 -- 172.21.15.83:0/1979877749 wait complete. 2024-09-14T03:40:18.040 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.036+0000 7ffa1f7bb640 1 Processor -- start 2024-09-14T03:40:18.040 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.036+0000 7ffa1f7bb640 1 -- start start 2024-09-14T03:40:18.040 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.036+0000 7ffa1f7bb640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7ffa18099080 0x7ffa18132920 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:18.040 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.036+0000 7ffa1f7bb640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7ffa180999a0 0x7ffa18132e60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:18.040 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.036+0000 7ffa1f7bb640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7ffa1812eff0 0x7ffa181333a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:18.040 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.036+0000 7ffa1f7bb640 1 -- --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7ffa181366d0 con 0x7ffa18099080 2024-09-14T03:40:18.040 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.037+0000 7ffa1f7bb640 1 -- --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7ffa18136550 con 0x7ffa180999a0 2024-09-14T03:40:18.041 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.037+0000 7ffa1f7bb640 1 -- --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7ffa18136850 con 0x7ffa1812eff0 2024-09-14T03:40:18.041 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.037+0000 7ffa1cd56640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7ffa180999a0 0x7ffa18132e60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:18.041 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.037+0000 7ffa1d557640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7ffa18099080 0x7ffa18132920 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:18.041 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.037+0000 7ffa1cd56640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7ffa180999a0 0x7ffa18132e60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3301/0 says I am v2:172.21.15.83:39184/0 (socket says 172.21.15.83:39184) 2024-09-14T03:40:18.041 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.037+0000 7ffa1cd56640 1 -- 172.21.15.83:0/1994392728 learned_addr learned my addr 172.21.15.83:0/1994392728 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:40:18.041 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.037+0000 7ffa1dd58640 1 --2- 172.21.15.83:0/1994392728 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7ffa1812eff0 0x7ffa181333a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:18.041 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.037+0000 7ffa1cd56640 1 -- 172.21.15.83:0/1994392728 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7ffa1812eff0 msgr2=0x7ffa181333a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:18.041 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.037+0000 7ffa1cd56640 1 --2- 172.21.15.83:0/1994392728 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7ffa1812eff0 0x7ffa181333a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:18.041 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.037+0000 7ffa1cd56640 1 -- 172.21.15.83:0/1994392728 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7ffa18099080 msgr2=0x7ffa18132920 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:18.041 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.037+0000 7ffa1cd56640 1 --2- 172.21.15.83:0/1994392728 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7ffa18099080 0x7ffa18132920 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:18.041 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.037+0000 7ffa1cd56640 1 -- 172.21.15.83:0/1994392728 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7ffa18074540 con 0x7ffa180999a0 2024-09-14T03:40:18.041 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.037+0000 7ffa1dd58640 1 --2- 172.21.15.83:0/1994392728 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7ffa1812eff0 0x7ffa181333a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:40:18.041 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.037+0000 7ffa1d557640 1 --2- 172.21.15.83:0/1994392728 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7ffa18099080 0x7ffa18132920 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:40:18.041 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.037+0000 7ffa1cd56640 1 --2- 172.21.15.83:0/1994392728 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7ffa180999a0 0x7ffa18132e60 secure :-1 s=READY pgs=14 cs=0 l=1 rev1=1 crypto rx=0x7ffa08009850 tx=0x7ffa08005e80 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:40:18.042 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.037+0000 7ffa0dffb640 1 -- 172.21.15.83:0/1994392728 <== mon.1 v2:172.21.15.83:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ffa0803f070 con 0x7ffa180999a0 2024-09-14T03:40:18.042 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.037+0000 7ffa0dffb640 1 -- 172.21.15.83:0/1994392728 <== mon.1 v2:172.21.15.83:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7ffa080047a0 con 0x7ffa180999a0 2024-09-14T03:40:18.042 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.037+0000 7ffa0dffb640 1 -- 172.21.15.83:0/1994392728 <== mon.1 v2:172.21.15.83:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ffa08035030 con 0x7ffa180999a0 2024-09-14T03:40:18.042 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.037+0000 7ffa1f7bb640 1 -- 172.21.15.83:0/1994392728 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7ffa18134090 con 0x7ffa180999a0 2024-09-14T03:40:18.042 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.038+0000 7ffa1f7bb640 1 -- 172.21.15.83:0/1994392728 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7ffa181338e0 con 0x7ffa180999a0 2024-09-14T03:40:18.042 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.039+0000 7ffa1f7bb640 1 -- 172.21.15.83:0/1994392728 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7ffa1812eab0 con 0x7ffa180999a0 2024-09-14T03:40:18.043 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.039+0000 7ffa0dffb640 1 -- 172.21.15.83:0/1994392728 <== mon.1 v2:172.21.15.83:3301/0 4 ==== mgrmap(e 5) ==== 55999+0+0 (secure 0 0 0) 0x7ffa08042070 con 0x7ffa180999a0 2024-09-14T03:40:18.043 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.039+0000 7ffa1d557640 1 -- 172.21.15.83:0/1994392728 >> v1:172.21.15.83:6800/1934165865 conn(0x7ff9fc047fd0 legacy=0x7ff9fc04a490 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v1:172.21.15.83:6800/1934165865 2024-09-14T03:40:18.043 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.039+0000 7ffa0dffb640 1 -- 172.21.15.83:0/1994392728 <== mon.1 v2:172.21.15.83:3301/0 5 ==== osd_map(20..20 src has 1..20) ==== 2812+0+0 (secure 0 0 0) 0x7ffa0807e570 con 0x7ffa180999a0 2024-09-14T03:40:18.045 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.043+0000 7ffa0dffb640 1 -- 172.21.15.83:0/1994392728 <== mon.1 v2:172.21.15.83:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+152684 (secure 0 0 0) 0x7ffa18074540 con 0x7ffa180999a0 2024-09-14T03:40:18.194 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.192+0000 7ffa1f7bb640 1 -- 172.21.15.83:0/1994392728 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) -- 0x7ffa180698c0 con 0x7ffa180999a0 2024-09-14T03:40:18.195 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.193+0000 7ffa0dffb640 1 -- 172.21.15.83:0/1994392728 <== mon.1 v2:172.21.15.83:3301/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v20) ==== 74+0+8828 (secure 0 0 0) 0x7ffa0807d700 con 0x7ffa180999a0 2024-09-14T03:40:18.195 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:40:18.195 INFO:teuthology.orchestra.run.smithi083.stdout:{"epoch":20,"fsid":"765298ef-df4c-4f28-af02-b45f60264ff9","created":"2024-09-14T03:38:43.954952+0000","modified":"2024-09-14T03:39:09.819316+0000","last_up_change":"2024-09-14T03:38:56.772236+0000","last_in_change":"2024-09-14T03:38:46.006167+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":4,"require_min_compat_client":"squid","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-14T03:38:56.922548+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"14","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":4,"score_stable":4,"optimal_score":0.5,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-14T03:39:00.433011+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":18,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.5,"score_stable":2.5,"optimal_score":1,"raw_score_acting":2.5,"raw_score_stable":2.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ff477614-343d-4d97-9df5-94933f9b904d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6805","nonce":2271402019}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6806","nonce":2271402019}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6808","nonce":2271402019}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6807","nonce":2271402019}]},"public_addr":"172.21.15.83:6805/2271402019","cluster_addr":"172.21.15.83:6806/2271402019","heartbeat_back_addr":"172.21.15.83:6808/2271402019","heartbeat_front_addr":"172.21.15.83:6807/2271402019","state":["exists","up"]},{"osd":1,"uuid":"562ec9a0-3f91-4a29-896d-94340e9fa36f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":11,"up_thru":15,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6801","nonce":2493296278}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6802","nonce":2493296278}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6804","nonce":2493296278}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6803","nonce":2493296278}]},"public_addr":"172.21.15.83:6801/2493296278","cluster_addr":"172.21.15.83:6802/2493296278","heartbeat_back_addr":"172.21.15.83:6804/2493296278","heartbeat_front_addr":"172.21.15.83:6803/2493296278","state":["exists","up"]},{"osd":2,"uuid":"8c242dda-3308-4804-b2be-9a1737c42cf6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":11,"up_thru":15,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6810","nonce":1553686396}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6812","nonce":1553686396}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6816","nonce":1553686396}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6814","nonce":1553686396}]},"public_addr":"172.21.15.83:6810/1553686396","cluster_addr":"172.21.15.83:6812/1553686396","heartbeat_back_addr":"172.21.15.83:6816/1553686396","heartbeat_front_addr":"172.21.15.83:6814/1553686396","state":["exists","up"]},{"osd":3,"uuid":"c4dd2c36-3c86-4b56-86fb-fede641a436c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":11,"up_thru":15,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6809","nonce":2350261690}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6811","nonce":2350261690}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6815","nonce":2350261690}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6813","nonce":2350261690}]},"public_addr":"172.21.15.83:6809/2350261690","cluster_addr":"172.21.15.83:6811/2350261690","heartbeat_back_addr":"172.21.15.83:6815/2350261690","heartbeat_front_addr":"172.21.15.83:6813/2350261690","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:38:53.047098+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:38:53.354351+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:38:53.422284+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:38:53.620657+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-14T03:40:18.197 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.195+0000 7ffa1f7bb640 1 -- 172.21.15.83:0/1994392728 >> v1:172.21.15.83:6800/1934165865 conn(0x7ff9fc047fd0 legacy=0x7ff9fc04a490 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-09-14T03:40:18.198 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.195+0000 7ffa1f7bb640 1 -- 172.21.15.83:0/1994392728 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7ffa180999a0 msgr2=0x7ffa18132e60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:18.198 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.195+0000 7ffa1f7bb640 1 --2- 172.21.15.83:0/1994392728 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7ffa180999a0 0x7ffa18132e60 secure :-1 s=READY pgs=14 cs=0 l=1 rev1=1 crypto rx=0x7ffa08009850 tx=0x7ffa08005e80 comp rx=0 tx=0).stop 2024-09-14T03:40:18.198 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.195+0000 7ffa1f7bb640 1 -- 172.21.15.83:0/1994392728 shutdown_connections 2024-09-14T03:40:18.198 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.195+0000 7ffa1f7bb640 1 --2- 172.21.15.83:0/1994392728 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7ffa1812eff0 0x7ffa181333a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:18.198 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.195+0000 7ffa1f7bb640 1 --2- 172.21.15.83:0/1994392728 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7ffa180999a0 0x7ffa18132e60 unknown :-1 s=CLOSED pgs=14 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:18.198 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.195+0000 7ffa1f7bb640 1 --2- 172.21.15.83:0/1994392728 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7ffa18099080 0x7ffa18132920 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:18.198 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.195+0000 7ffa1f7bb640 1 -- 172.21.15.83:0/1994392728 >> 172.21.15.83:0/1994392728 conn(0x7ffa180a1800 msgr2=0x7ffa181264a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:18.198 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.195+0000 7ffa1f7bb640 1 -- 172.21.15.83:0/1994392728 shutdown_connections 2024-09-14T03:40:18.198 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.195+0000 7ffa1f7bb640 1 -- 172.21.15.83:0/1994392728 wait complete. 2024-09-14T03:40:18.209 INFO:tasks.ceph.ceph_manager.ceph:all up! 2024-09-14T03:40:18.209 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-09-14T03:40:18.312 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.310+0000 7f4d9a639640 1 -- 172.21.15.83:0/1609245908 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f4d94099080 msgr2=0x7f4d94099460 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:18.313 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.310+0000 7f4d9a639640 1 --2- 172.21.15.83:0/1609245908 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f4d94099080 0x7f4d94099460 secure :-1 s=READY pgs=20 cs=0 l=1 rev1=1 crypto rx=0x7f4d84009830 tx=0x7f4d8402a480 comp rx=0 tx=0).stop 2024-09-14T03:40:18.313 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.310+0000 7f4d9a639640 1 -- 172.21.15.83:0/1609245908 shutdown_connections 2024-09-14T03:40:18.313 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.310+0000 7f4d9a639640 1 --2- 172.21.15.83:0/1609245908 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f4d9412f1c0 0x7f4d9412f610 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:18.313 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.310+0000 7f4d9a639640 1 --2- 172.21.15.83:0/1609245908 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f4d94099080 0x7f4d94099460 unknown :-1 s=CLOSED pgs=20 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:18.313 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.310+0000 7f4d9a639640 1 -- 172.21.15.83:0/1609245908 >> 172.21.15.83:0/1609245908 conn(0x7f4d940a1800 msgr2=0x7f4d940a1c10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:18.313 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.310+0000 7f4d9a639640 1 -- 172.21.15.83:0/1609245908 shutdown_connections 2024-09-14T03:40:18.313 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.310+0000 7f4d9a639640 1 -- 172.21.15.83:0/1609245908 wait complete. 2024-09-14T03:40:18.315 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.311+0000 7f4d9a639640 1 Processor -- start 2024-09-14T03:40:18.315 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.311+0000 7f4d9a639640 1 -- start start 2024-09-14T03:40:18.315 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.311+0000 7f4d9a639640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f4d94099080 0x7f4d940957f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:18.315 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.311+0000 7f4d9a639640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f4d940999a0 0x7f4d94095d30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:18.316 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.311+0000 7f4d9a639640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f4d9412f1c0 0x7f4d94096270 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:18.316 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.311+0000 7f4d9a639640 1 -- --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7f4d94137660 con 0x7f4d9412f1c0 2024-09-14T03:40:18.316 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.311+0000 7f4d9a639640 1 -- --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7f4d941374e0 con 0x7f4d94099080 2024-09-14T03:40:18.316 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.311+0000 7f4d9a639640 1 -- --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7f4d941377e0 con 0x7f4d940999a0 2024-09-14T03:40:18.316 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.311+0000 7f4d93fff640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f4d94099080 0x7f4d940957f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:18.316 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.311+0000 7f4d93fff640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f4d94099080 0x7f4d940957f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3301/0 says I am v2:172.21.15.83:39194/0 (socket says 172.21.15.83:39194) 2024-09-14T03:40:18.316 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.311+0000 7f4d93fff640 1 -- 172.21.15.83:0/2131650620 learned_addr learned my addr 172.21.15.83:0/2131650620 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:40:18.316 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.311+0000 7f4d98bd6640 1 --2- 172.21.15.83:0/2131650620 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f4d9412f1c0 0x7f4d94096270 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:18.316 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.311+0000 7f4d937fe640 1 --2- 172.21.15.83:0/2131650620 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f4d940999a0 0x7f4d94095d30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:18.316 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.312+0000 7f4d98bd6640 1 -- 172.21.15.83:0/2131650620 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f4d940999a0 msgr2=0x7f4d94095d30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:18.316 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.312+0000 7f4d98bd6640 1 --2- 172.21.15.83:0/2131650620 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f4d940999a0 0x7f4d94095d30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:18.316 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.312+0000 7f4d98bd6640 1 -- 172.21.15.83:0/2131650620 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f4d94099080 msgr2=0x7f4d940957f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:18.316 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.312+0000 7f4d98bd6640 1 --2- 172.21.15.83:0/2131650620 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f4d94099080 0x7f4d940957f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:18.316 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.312+0000 7f4d98bd6640 1 -- 172.21.15.83:0/2131650620 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f4d940938a0 con 0x7f4d9412f1c0 2024-09-14T03:40:18.317 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.312+0000 7f4d93fff640 1 --2- 172.21.15.83:0/2131650620 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f4d94099080 0x7f4d940957f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:40:18.317 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.312+0000 7f4d98bd6640 1 --2- 172.21.15.83:0/2131650620 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f4d9412f1c0 0x7f4d94096270 secure :-1 s=READY pgs=21 cs=0 l=1 rev1=1 crypto rx=0x7f4d8c00ec50 tx=0x7f4d8c00c6a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:40:18.317 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.312+0000 7f4d90ff9640 1 -- 172.21.15.83:0/2131650620 <== mon.0 v2:172.21.15.83:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4d8c003b70 con 0x7f4d9412f1c0 2024-09-14T03:40:18.317 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.312+0000 7f4d90ff9640 1 -- 172.21.15.83:0/2131650620 <== mon.0 v2:172.21.15.83:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f4d8c003d10 con 0x7f4d9412f1c0 2024-09-14T03:40:18.317 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.312+0000 7f4d9a639640 1 -- 172.21.15.83:0/2131650620 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f4d941294d0 con 0x7f4d9412f1c0 2024-09-14T03:40:18.317 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.312+0000 7f4d90ff9640 1 -- 172.21.15.83:0/2131650620 <== mon.0 v2:172.21.15.83:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4d8c004280 con 0x7f4d9412f1c0 2024-09-14T03:40:18.317 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.312+0000 7f4d9a639640 1 -- 172.21.15.83:0/2131650620 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f4d94097110 con 0x7f4d9412f1c0 2024-09-14T03:40:18.318 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.313+0000 7f4d90ff9640 1 -- 172.21.15.83:0/2131650620 <== mon.0 v2:172.21.15.83:3300/0 4 ==== mgrmap(e 5) ==== 55999+0+0 (secure 0 0 0) 0x7f4d8c004420 con 0x7f4d9412f1c0 2024-09-14T03:40:18.318 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.313+0000 7f4d9a639640 1 -- 172.21.15.83:0/2131650620 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f4d94099460 con 0x7f4d9412f1c0 2024-09-14T03:40:18.318 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.314+0000 7f4d93fff640 1 -- 172.21.15.83:0/2131650620 >> v1:172.21.15.83:6800/1934165865 conn(0x7f4d80047fd0 legacy=0x7f4d8004a490 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v1:172.21.15.83:6800/1934165865 2024-09-14T03:40:18.318 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.314+0000 7f4d90ff9640 1 -- 172.21.15.83:0/2131650620 <== mon.0 v2:172.21.15.83:3300/0 5 ==== osd_map(20..20 src has 1..20) ==== 2812+0+0 (secure 0 0 0) 0x7f4d8c05e4f0 con 0x7f4d9412f1c0 2024-09-14T03:40:18.320 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.318+0000 7f4d90ff9640 1 -- 172.21.15.83:0/2131650620 <== mon.0 v2:172.21.15.83:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+152684 (secure 0 0 0) 0x7f4d8c05d680 con 0x7f4d9412f1c0 2024-09-14T03:40:18.468 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.465+0000 7f4d9a639640 1 -- 172.21.15.83:0/2131650620 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) -- 0x7f4d940698c0 con 0x7f4d9412f1c0 2024-09-14T03:40:18.468 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.466+0000 7f4d90ff9640 1 -- 172.21.15.83:0/2131650620 <== mon.0 v2:172.21.15.83:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v20) ==== 74+0+8828 (secure 0 0 0) 0x7f4d8c03d850 con 0x7f4d9412f1c0 2024-09-14T03:40:18.469 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:40:18.469 INFO:teuthology.orchestra.run.smithi083.stdout:{"epoch":20,"fsid":"765298ef-df4c-4f28-af02-b45f60264ff9","created":"2024-09-14T03:38:43.954952+0000","modified":"2024-09-14T03:39:09.819316+0000","last_up_change":"2024-09-14T03:38:56.772236+0000","last_in_change":"2024-09-14T03:38:46.006167+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":4,"require_min_compat_client":"squid","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-14T03:38:56.922548+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"14","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":4,"score_stable":4,"optimal_score":0.5,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-14T03:39:00.433011+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":18,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.5,"score_stable":2.5,"optimal_score":1,"raw_score_acting":2.5,"raw_score_stable":2.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ff477614-343d-4d97-9df5-94933f9b904d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6805","nonce":2271402019}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6806","nonce":2271402019}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6808","nonce":2271402019}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6807","nonce":2271402019}]},"public_addr":"172.21.15.83:6805/2271402019","cluster_addr":"172.21.15.83:6806/2271402019","heartbeat_back_addr":"172.21.15.83:6808/2271402019","heartbeat_front_addr":"172.21.15.83:6807/2271402019","state":["exists","up"]},{"osd":1,"uuid":"562ec9a0-3f91-4a29-896d-94340e9fa36f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":11,"up_thru":15,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6801","nonce":2493296278}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6802","nonce":2493296278}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6804","nonce":2493296278}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6803","nonce":2493296278}]},"public_addr":"172.21.15.83:6801/2493296278","cluster_addr":"172.21.15.83:6802/2493296278","heartbeat_back_addr":"172.21.15.83:6804/2493296278","heartbeat_front_addr":"172.21.15.83:6803/2493296278","state":["exists","up"]},{"osd":2,"uuid":"8c242dda-3308-4804-b2be-9a1737c42cf6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":11,"up_thru":15,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6810","nonce":1553686396}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6812","nonce":1553686396}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6816","nonce":1553686396}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6814","nonce":1553686396}]},"public_addr":"172.21.15.83:6810/1553686396","cluster_addr":"172.21.15.83:6812/1553686396","heartbeat_back_addr":"172.21.15.83:6816/1553686396","heartbeat_front_addr":"172.21.15.83:6814/1553686396","state":["exists","up"]},{"osd":3,"uuid":"c4dd2c36-3c86-4b56-86fb-fede641a436c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":11,"up_thru":15,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6809","nonce":2350261690}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6811","nonce":2350261690}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6815","nonce":2350261690}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6813","nonce":2350261690}]},"public_addr":"172.21.15.83:6809/2350261690","cluster_addr":"172.21.15.83:6811/2350261690","heartbeat_back_addr":"172.21.15.83:6815/2350261690","heartbeat_front_addr":"172.21.15.83:6813/2350261690","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:38:53.047098+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:38:53.354351+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:38:53.422284+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:38:53.620657+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-14T03:40:18.471 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.468+0000 7f4d9a639640 1 -- 172.21.15.83:0/2131650620 >> v1:172.21.15.83:6800/1934165865 conn(0x7f4d80047fd0 legacy=0x7f4d8004a490 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-09-14T03:40:18.471 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.468+0000 7f4d9a639640 1 -- 172.21.15.83:0/2131650620 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f4d9412f1c0 msgr2=0x7f4d94096270 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:18.471 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.468+0000 7f4d9a639640 1 --2- 172.21.15.83:0/2131650620 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f4d9412f1c0 0x7f4d94096270 secure :-1 s=READY pgs=21 cs=0 l=1 rev1=1 crypto rx=0x7f4d8c00ec50 tx=0x7f4d8c00c6a0 comp rx=0 tx=0).stop 2024-09-14T03:40:18.471 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.468+0000 7f4d9a639640 1 -- 172.21.15.83:0/2131650620 shutdown_connections 2024-09-14T03:40:18.471 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.468+0000 7f4d9a639640 1 --2- 172.21.15.83:0/2131650620 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f4d9412f1c0 0x7f4d94096270 unknown :-1 s=CLOSED pgs=21 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:18.471 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.468+0000 7f4d9a639640 1 --2- 172.21.15.83:0/2131650620 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f4d940999a0 0x7f4d94095d30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:18.471 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.468+0000 7f4d9a639640 1 --2- 172.21.15.83:0/2131650620 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f4d94099080 0x7f4d940957f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:18.471 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.468+0000 7f4d9a639640 1 -- 172.21.15.83:0/2131650620 >> 172.21.15.83:0/2131650620 conn(0x7f4d940a1800 msgr2=0x7f4d94097940 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:18.472 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.468+0000 7f4d9a639640 1 -- 172.21.15.83:0/2131650620 shutdown_connections 2024-09-14T03:40:18.472 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.468+0000 7f4d9a639640 1 -- 172.21.15.83:0/2131650620 wait complete. 2024-09-14T03:40:18.483 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.0 flush_pg_stats 2024-09-14T03:40:18.483 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.1 flush_pg_stats 2024-09-14T03:40:18.484 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.2 flush_pg_stats 2024-09-14T03:40:18.484 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.3 flush_pg_stats 2024-09-14T03:40:18.648 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.645+0000 7fb0de2ef640 1 -- 172.21.15.83:0/2909688456 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fb0d812fae0 msgr2=0x7fb0d8131ed0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:18.648 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.645+0000 7fb0de2ef640 1 --2- 172.21.15.83:0/2909688456 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fb0d812fae0 0x7fb0d8131ed0 secure :-1 s=READY pgs=18 cs=0 l=1 rev1=1 crypto rx=0x7fb0c8005fd0 tx=0x7fb0c80329c0 comp rx=0 tx=0).stop 2024-09-14T03:40:18.648 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.645+0000 7fb0de2ef640 1 -- 172.21.15.83:0/2909688456 shutdown_connections 2024-09-14T03:40:18.649 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.645+0000 7fb0de2ef640 1 --2- 172.21.15.83:0/2909688456 >> v2:172.21.15.83:3300/0 conn(0x7fb0d8132410 0x7fb0d8134800 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:18.649 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.645+0000 7fb0de2ef640 1 --2- 172.21.15.83:0/2909688456 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fb0d812fae0 0x7fb0d8131ed0 unknown :-1 s=CLOSED pgs=18 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:18.649 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.645+0000 7fb0de2ef640 1 -- 172.21.15.83:0/2909688456 >> 172.21.15.83:0/2909688456 conn(0x7fb0d809d480 msgr2=0x7fb0d809d890 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:18.649 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.645+0000 7fb0de2ef640 1 -- 172.21.15.83:0/2909688456 shutdown_connections 2024-09-14T03:40:18.649 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.645+0000 7fb0de2ef640 1 -- 172.21.15.83:0/2909688456 wait complete. 2024-09-14T03:40:18.650 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.646+0000 7fa8217bb640 1 -- 172.21.15.83:0/1894877180 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fa81c092820 msgr2=0x7fa81c092c00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:18.650 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.646+0000 7fa8217bb640 1 --2- 172.21.15.83:0/1894877180 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fa81c092820 0x7fa81c092c00 secure :-1 s=READY pgs=23 cs=0 l=1 rev1=1 crypto rx=0x7fa80c009860 tx=0x7fa80c02a390 comp rx=0 tx=0).stop 2024-09-14T03:40:18.650 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.646+0000 7fa8217bb640 1 -- 172.21.15.83:0/1894877180 shutdown_connections 2024-09-14T03:40:18.650 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.646+0000 7fa8217bb640 1 --2- 172.21.15.83:0/1894877180 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fa81c12bf30 0x7fa81c12c350 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:18.650 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.646+0000 7fa8217bb640 1 --2- 172.21.15.83:0/1894877180 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fa81c092820 0x7fa81c092c00 unknown :-1 s=CLOSED pgs=23 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:18.650 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.646+0000 7fa8217bb640 1 -- 172.21.15.83:0/1894877180 >> 172.21.15.83:0/1894877180 conn(0x7fa81c09e490 msgr2=0x7fa81c09e8a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:18.650 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.646+0000 7fb0de2ef640 1 Processor -- start 2024-09-14T03:40:18.651 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.647+0000 7fa8217bb640 1 -- 172.21.15.83:0/1894877180 shutdown_connections 2024-09-14T03:40:18.651 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.647+0000 7fa8217bb640 1 -- 172.21.15.83:0/1894877180 wait complete. 2024-09-14T03:40:18.652 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.647+0000 7fb0de2ef640 1 -- start start 2024-09-14T03:40:18.652 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.647+0000 7fb0de2ef640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fb0d8129460 0x7fb0d81226b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:18.652 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.647+0000 7fb0de2ef640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fb0d8132410 0x7fb0d8122bf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:18.652 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.647+0000 7fb0de2ef640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fb0d812a5c0 0x7fb0d812a9d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:18.652 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.647+0000 7fb0de2ef640 1 -- --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7fb0d806fd60 con 0x7fb0d8132410 2024-09-14T03:40:18.652 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.647+0000 7fb0de2ef640 1 -- --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7fb0d806fbe0 con 0x7fb0d812a5c0 2024-09-14T03:40:18.652 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.647+0000 7fb0de2ef640 1 -- --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7fb0d806fee0 con 0x7fb0d8129460 2024-09-14T03:40:18.652 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.647+0000 7fb0d77fe640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fb0d8132410 0x7fb0d8122bf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:18.652 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.647+0000 7f8521228640 1 -- 172.21.15.83:0/1260609610 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f851c0952e0 msgr2=0x7f851c133280 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:18.653 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.647+0000 7f8521228640 1 --2- 172.21.15.83:0/1260609610 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f851c0952e0 0x7f851c133280 secure :-1 s=READY pgs=24 cs=0 l=1 rev1=1 crypto rx=0x7f8504005fd0 tx=0x7f850402a420 comp rx=0 tx=0).stop 2024-09-14T03:40:18.653 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.647+0000 7fb0d7fff640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fb0d8129460 0x7fb0d81226b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:18.653 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.648+0000 7f8521228640 1 -- 172.21.15.83:0/1260609610 shutdown_connections 2024-09-14T03:40:18.653 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.648+0000 7f8521228640 1 --2- 172.21.15.83:0/1260609610 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f851c0956c0 0x7f851c1357d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:18.653 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.648+0000 7f8521228640 1 --2- 172.21.15.83:0/1260609610 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f851c0952e0 0x7f851c133280 unknown :-1 s=CLOSED pgs=24 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:18.653 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.648+0000 7f8521228640 1 --2- 172.21.15.83:0/1260609610 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f851c0949c0 0x7f851c094da0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:18.654 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.648+0000 7f8521228640 1 -- 172.21.15.83:0/1260609610 >> 172.21.15.83:0/1260609610 conn(0x7f851c09d4f0 msgr2=0x7f851c09d900 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:18.654 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.648+0000 7fb0dc88c640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fb0d812a5c0 0x7fb0d812a9d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:18.654 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.648+0000 7fb0d7fff640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fb0d8129460 0x7fb0d81226b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3302/0 says I am v2:172.21.15.83:58342/0 (socket says 172.21.15.83:58342) 2024-09-14T03:40:18.654 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.648+0000 7fb0d7fff640 1 -- 172.21.15.83:0/463394540 learned_addr learned my addr 172.21.15.83:0/463394540 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:40:18.655 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.648+0000 7fa8217bb640 1 Processor -- start 2024-09-14T03:40:18.655 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.648+0000 7fa8217bb640 1 -- start start 2024-09-14T03:40:18.655 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.648+0000 7fb0dc88c640 1 -- 172.21.15.83:0/463394540 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fb0d8129460 msgr2=0x7fb0d81226b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:18.655 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.648+0000 7fb0dc88c640 1 --2- 172.21.15.83:0/463394540 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fb0d8129460 0x7fb0d81226b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:18.656 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.648+0000 7fb0dc88c640 1 -- 172.21.15.83:0/463394540 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fb0d8132410 msgr2=0x7fb0d8122bf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:18.656 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.648+0000 7fb0dc88c640 1 --2- 172.21.15.83:0/463394540 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fb0d8132410 0x7fb0d8122bf0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:18.656 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.648+0000 7fb0dc88c640 1 -- 172.21.15.83:0/463394540 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fb0d812d010 con 0x7fb0d812a5c0 2024-09-14T03:40:18.656 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.648+0000 7fb0d77fe640 1 --2- 172.21.15.83:0/463394540 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fb0d8132410 0x7fb0d8122bf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-09-14T03:40:18.656 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.648+0000 7fb0d7fff640 1 --2- 172.21.15.83:0/463394540 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fb0d8129460 0x7fb0d81226b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-09-14T03:40:18.656 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.648+0000 7f8521228640 1 -- 172.21.15.83:0/1260609610 shutdown_connections 2024-09-14T03:40:18.657 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.648+0000 7f8521228640 1 -- 172.21.15.83:0/1260609610 wait complete. 2024-09-14T03:40:18.657 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.649+0000 7fa8217bb640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fa81c092820 0x7fa81c06f710 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:18.657 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.649+0000 7fa8217bb640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fa81c093140 0x7fa81c06fc50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:18.657 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.649+0000 7fa8217bb640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fa81c12bf30 0x7fa81c070190 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:18.657 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.649+0000 7fa8217bb640 1 -- --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7fa81c06ed50 con 0x7fa81c093140 2024-09-14T03:40:18.657 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.649+0000 7fa8217bb640 1 -- --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7fa81c06ebd0 con 0x7fa81c12bf30 2024-09-14T03:40:18.658 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.649+0000 7fa8217bb640 1 -- --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7fa81c06eed0 con 0x7fa81c092820 2024-09-14T03:40:18.658 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.648+0000 7fb0dc88c640 1 --2- 172.21.15.83:0/463394540 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fb0d812a5c0 0x7fb0d812a9d0 secure :-1 s=READY pgs=15 cs=0 l=1 rev1=1 crypto rx=0x7fb0d000be00 tx=0x7fb0d000bef0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:40:18.658 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.649+0000 7fb0d4ff9640 1 -- 172.21.15.83:0/463394540 <== mon.1 v2:172.21.15.83:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb0d0019070 con 0x7fb0d812a5c0 2024-09-14T03:40:18.658 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.649+0000 7fb0d4ff9640 1 -- 172.21.15.83:0/463394540 <== mon.1 v2:172.21.15.83:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fb0d0014440 con 0x7fb0d812a5c0 2024-09-14T03:40:18.658 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.649+0000 7fb0d4ff9640 1 -- 172.21.15.83:0/463394540 <== mon.1 v2:172.21.15.83:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb0d00148d0 con 0x7fb0d812a5c0 2024-09-14T03:40:18.659 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.649+0000 7fa81b7fe640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fa81c12bf30 0x7fa81c070190 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:18.659 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.649+0000 7fa81b7fe640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fa81c12bf30 0x7fa81c070190 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3301/0 says I am v2:172.21.15.83:39204/0 (socket says 172.21.15.83:39204) 2024-09-14T03:40:18.659 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.649+0000 7fa81b7fe640 1 -- 172.21.15.83:0/3822169017 learned_addr learned my addr 172.21.15.83:0/3822169017 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:40:18.659 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.649+0000 7fa81a7fc640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fa81c093140 0x7fa81c06fc50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:18.660 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.649+0000 7fa81affd640 1 --2- 172.21.15.83:0/3822169017 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fa81c092820 0x7fa81c06f710 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:18.660 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.650+0000 7fa81b7fe640 1 -- 172.21.15.83:0/3822169017 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fa81c092820 msgr2=0x7fa81c06f710 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:18.660 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.650+0000 7fa81b7fe640 1 --2- 172.21.15.83:0/3822169017 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fa81c092820 0x7fa81c06f710 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:18.660 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.650+0000 7fa81b7fe640 1 -- 172.21.15.83:0/3822169017 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fa81c093140 msgr2=0x7fa81c06fc50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:18.660 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.650+0000 7fa81b7fe640 1 --2- 172.21.15.83:0/3822169017 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fa81c093140 0x7fa81c06fc50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:18.660 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.650+0000 7fa81b7fe640 1 -- 172.21.15.83:0/3822169017 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fa81c0745e0 con 0x7fa81c12bf30 2024-09-14T03:40:18.660 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.650+0000 7fa81a7fc640 1 --2- 172.21.15.83:0/3822169017 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fa81c093140 0x7fa81c06fc50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:40:18.660 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.649+0000 7fb0de2ef640 1 -- 172.21.15.83:0/463394540 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fb0d8126ec0 con 0x7fb0d812a5c0 2024-09-14T03:40:18.660 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.649+0000 7fb0de2ef640 1 -- 172.21.15.83:0/463394540 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7fb0d8123130 con 0x7fb0d812a5c0 2024-09-14T03:40:18.661 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.650+0000 7f8521228640 1 Processor -- start 2024-09-14T03:40:18.661 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.650+0000 7f48d611a640 1 -- 172.21.15.83:0/4185044759 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f48d012f330 msgr2=0x7f48d0137090 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:18.661 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.650+0000 7f48d611a640 1 --2- 172.21.15.83:0/4185044759 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f48d012f330 0x7f48d0137090 secure :-1 s=READY pgs=25 cs=0 l=1 rev1=1 crypto rx=0x7f48c8005fd0 tx=0x7f48c802a6f0 comp rx=0 tx=0).stop 2024-09-14T03:40:18.662 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.650+0000 7fa81b7fe640 1 --2- 172.21.15.83:0/3822169017 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fa81c12bf30 0x7fa81c070190 secure :-1 s=READY pgs=16 cs=0 l=1 rev1=1 crypto rx=0x7fa81400df70 tx=0x7fa81400b630 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:40:18.662 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.650+0000 7f48d611a640 1 -- 172.21.15.83:0/4185044759 shutdown_connections 2024-09-14T03:40:18.662 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.650+0000 7f48d611a640 1 --2- 172.21.15.83:0/4185044759 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f48d012f330 0x7f48d0137090 unknown :-1 s=CLOSED pgs=25 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:18.662 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.650+0000 7f48d611a640 1 --2- 172.21.15.83:0/4185044759 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f48d00700b0 0x7f48d0070490 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:18.662 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.650+0000 7f48d611a640 1 -- 172.21.15.83:0/4185044759 >> 172.21.15.83:0/4185044759 conn(0x7f48d009e0d0 msgr2=0x7f48d009a5a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:18.663 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.650+0000 7fa8037fe640 1 -- 172.21.15.83:0/3822169017 <== mon.1 v2:172.21.15.83:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa814003970 con 0x7fa81c12bf30 2024-09-14T03:40:18.663 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.651+0000 7fa8217bb640 1 -- 172.21.15.83:0/3822169017 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fa81c072990 con 0x7fa81c12bf30 2024-09-14T03:40:18.663 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.651+0000 7fa8217bb640 1 -- 172.21.15.83:0/3822169017 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7fa81c072c50 con 0x7fa81c12bf30 2024-09-14T03:40:18.663 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.651+0000 7fb0d4ff9640 1 -- 172.21.15.83:0/463394540 <== mon.1 v2:172.21.15.83:3301/0 4 ==== mgrmap(e 5) ==== 55999+0+0 (secure 0 0 0) 0x7fb0d001d3f0 con 0x7fb0d812a5c0 2024-09-14T03:40:18.663 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.651+0000 7f48d611a640 1 -- 172.21.15.83:0/4185044759 shutdown_connections 2024-09-14T03:40:18.663 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.651+0000 7f48d611a640 1 -- 172.21.15.83:0/4185044759 wait complete. 2024-09-14T03:40:18.664 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.652+0000 7fa8037fe640 1 -- 172.21.15.83:0/3822169017 <== mon.1 v2:172.21.15.83:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fa814003b10 con 0x7fa81c12bf30 2024-09-14T03:40:18.664 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.652+0000 7fa8037fe640 1 -- 172.21.15.83:0/3822169017 <== mon.1 v2:172.21.15.83:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa814004020 con 0x7fa81c12bf30 2024-09-14T03:40:18.665 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.651+0000 7f8521228640 1 -- start start 2024-09-14T03:40:18.665 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.651+0000 7f8521228640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f851c0949c0 0x7f851c1350e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:18.665 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.651+0000 7f8521228640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f851c0952e0 0x7f851c135620 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:18.665 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.651+0000 7f8521228640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f851c0956c0 0x7f851c08bfb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:18.666 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.651+0000 7f8521228640 1 -- --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7f851c136830 con 0x7f851c0956c0 2024-09-14T03:40:18.666 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.651+0000 7f8521228640 1 -- --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7f851c1366b0 con 0x7f851c0949c0 2024-09-14T03:40:18.666 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.651+0000 7f8521228640 1 -- --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7f851c1369b0 con 0x7f851c0952e0 2024-09-14T03:40:18.666 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.651+0000 7f851ad9d640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f851c0949c0 0x7f851c1350e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:18.666 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.652+0000 7f851ad9d640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f851c0949c0 0x7f851c1350e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3301/0 says I am v2:172.21.15.83:39220/0 (socket says 172.21.15.83:39220) 2024-09-14T03:40:18.666 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.652+0000 7f48d611a640 1 Processor -- start 2024-09-14T03:40:18.666 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.652+0000 7f851ad9d640 1 -- 172.21.15.83:0/2656368403 learned_addr learned my addr 172.21.15.83:0/2656368403 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:40:18.667 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.652+0000 7f851ad9d640 1 -- 172.21.15.83:0/2656368403 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f851c0952e0 msgr2=0x7f851c135620 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-09-14T03:40:18.667 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.652+0000 7fb0d7fff640 1 -- 172.21.15.83:0/463394540 >> v1:172.21.15.83:6800/1934165865 conn(0x7fb0b8048050 legacy=0x7fb0b804a510 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v1:172.21.15.83:6800/1934165865 2024-09-14T03:40:18.667 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.652+0000 7fb0de2ef640 1 -- 172.21.15.83:0/463394540 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_get_version(what=osdmap handle=1) -- 0x7fb0d8129840 con 0x7fb0d812a5c0 2024-09-14T03:40:18.667 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.652+0000 7fb0d4ff9640 1 -- 172.21.15.83:0/463394540 <== mon.1 v2:172.21.15.83:3301/0 5 ==== osd_map(20..20 src has 1..20) ==== 2812+0+0 (secure 0 0 0) 0x7fb0d005d860 con 0x7fb0d812a5c0 2024-09-14T03:40:18.667 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.652+0000 7fb0d4ff9640 1 -- 172.21.15.83:0/463394540 --> v1:172.21.15.83:6805/2271402019 -- command(tid 1: {"prefix": "get_command_descriptions"}) -- 0x7fb0d000de50 con 0x7fb0b804e740 2024-09-14T03:40:18.667 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.653+0000 7fb0d4ff9640 1 -- 172.21.15.83:0/463394540 <== mon.1 v2:172.21.15.83:3301/0 6 ==== mon_get_version_reply(handle=1 version=20) ==== 24+0+0 (secure 0 0 0) 0x7fb0d003aae0 con 0x7fb0d812a5c0 2024-09-14T03:40:18.668 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.653+0000 7f851a59c640 1 --2- 172.21.15.83:0/2656368403 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f851c0952e0 0x7f851c135620 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:18.669 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.653+0000 7f851b59e640 1 --2- 172.21.15.83:0/2656368403 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f851c0956c0 0x7f851c08bfb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:18.669 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.653+0000 7f851ad9d640 1 --2- 172.21.15.83:0/2656368403 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f851c0952e0 0x7f851c135620 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:18.669 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.653+0000 7f851ad9d640 1 -- 172.21.15.83:0/2656368403 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f851c0956c0 msgr2=0x7f851c08bfb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:18.669 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.653+0000 7f851ad9d640 1 --2- 172.21.15.83:0/2656368403 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f851c0956c0 0x7f851c08bfb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:18.669 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.653+0000 7f851ad9d640 1 -- 172.21.15.83:0/2656368403 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f851c0747d0 con 0x7f851c0949c0 2024-09-14T03:40:18.669 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.653+0000 7f851b59e640 1 --2- 172.21.15.83:0/2656368403 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f851c0956c0 0x7f851c08bfb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-09-14T03:40:18.669 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.653+0000 7f851a59c640 1 --2- 172.21.15.83:0/2656368403 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f851c0952e0 0x7f851c135620 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-09-14T03:40:18.669 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.653+0000 7f851ad9d640 1 --2- 172.21.15.83:0/2656368403 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f851c0949c0 0x7f851c1350e0 secure :-1 s=READY pgs=17 cs=0 l=1 rev1=1 crypto rx=0x7f851400ea90 tx=0x7f851400ef50 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:40:18.669 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.653+0000 7f85137fe640 1 -- 172.21.15.83:0/2656368403 <== mon.1 v2:172.21.15.83:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f851400ce40 con 0x7f851c0949c0 2024-09-14T03:40:18.669 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.653+0000 7f85137fe640 1 -- 172.21.15.83:0/2656368403 <== mon.1 v2:172.21.15.83:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f8514004020 con 0x7f851c0949c0 2024-09-14T03:40:18.669 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.654+0000 7f8521228640 1 -- 172.21.15.83:0/2656368403 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f851c08f890 con 0x7f851c0949c0 2024-09-14T03:40:18.669 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.654+0000 7f8521228640 1 -- 172.21.15.83:0/2656368403 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7f851c08c4f0 con 0x7f851c0949c0 2024-09-14T03:40:18.670 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.654+0000 7fa8217bb640 1 -- 172.21.15.83:0/3822169017 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_get_version(what=osdmap handle=1) -- 0x7fa7fc001030 con 0x7fa81c12bf30 2024-09-14T03:40:18.670 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.654+0000 7f48d611a640 1 -- start start 2024-09-14T03:40:18.670 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.654+0000 7f48d611a640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f48d00700b0 0x7f48d0096a30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:18.670 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.654+0000 7f85137fe640 1 -- 172.21.15.83:0/2656368403 <== mon.1 v2:172.21.15.83:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f851400ce40 con 0x7f851c0949c0 2024-09-14T03:40:18.671 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.655+0000 7f48d611a640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f48d00709d0 0x7f48d0096f70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:18.671 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.655+0000 7f48d611a640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f48d012f330 0x7f48d012b370 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:18.671 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.655+0000 7f48c6ffd640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f48d00709d0 0x7f48d0096f70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:18.671 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.655+0000 7f48c6ffd640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f48d00709d0 0x7f48d0096f70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3302/0 says I am v2:172.21.15.83:58364/0 (socket says 172.21.15.83:58364) 2024-09-14T03:40:18.672 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.655+0000 7f8521228640 1 -- 172.21.15.83:0/2656368403 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_get_version(what=osdmap handle=1) -- 0x7f851c135b60 con 0x7f851c0949c0 2024-09-14T03:40:18.672 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.655+0000 7fa8037fe640 1 -- 172.21.15.83:0/3822169017 <== mon.1 v2:172.21.15.83:3301/0 4 ==== mgrmap(e 5) ==== 55999+0+0 (secure 0 0 0) 0x7fa81401e020 con 0x7fa81c12bf30 2024-09-14T03:40:18.673 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.655+0000 7f48d611a640 1 -- --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7f48d01379b0 con 0x7f48d00700b0 2024-09-14T03:40:18.673 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.655+0000 7f48d611a640 1 -- --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7f48d0137830 con 0x7f48d012f330 2024-09-14T03:40:18.673 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.655+0000 7f48d611a640 1 -- --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7f48d0137b30 con 0x7f48d00709d0 2024-09-14T03:40:18.673 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.655+0000 7f48c6ffd640 1 -- 172.21.15.83:0/2653415622 learned_addr learned my addr 172.21.15.83:0/2653415622 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:40:18.673 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.655+0000 7f48cf7fe640 1 --2- 172.21.15.83:0/2653415622 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f48d00700b0 0x7f48d0096a30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:18.673 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.656+0000 7f48c6ffd640 1 -- 172.21.15.83:0/2653415622 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f48d012f330 msgr2=0x7f48d012b370 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:18.673 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.656+0000 7f48cffff640 1 --2- 172.21.15.83:0/2653415622 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f48d012f330 0x7f48d012b370 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:18.673 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.656+0000 7f48c6ffd640 1 --2- 172.21.15.83:0/2653415622 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f48d012f330 0x7f48d012b370 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:18.674 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.656+0000 7fa8037fe640 1 -- 172.21.15.83:0/3822169017 <== mon.1 v2:172.21.15.83:3301/0 5 ==== osd_map(20..20 src has 1..20) ==== 2812+0+0 (secure 0 0 0) 0x7fa81405dac0 con 0x7fa81c12bf30 2024-09-14T03:40:18.674 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.656+0000 7fa8037fe640 1 -- 172.21.15.83:0/3822169017 --> v1:172.21.15.83:6809/2350261690 -- command(tid 1: {"prefix": "get_command_descriptions"}) -- 0x7fa814019e50 con 0x7fa80804e650 2024-09-14T03:40:18.674 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.656+0000 7f48c6ffd640 1 -- 172.21.15.83:0/2653415622 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f48d00700b0 msgr2=0x7f48d0096a30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:18.674 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.656+0000 7f48c6ffd640 1 --2- 172.21.15.83:0/2653415622 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f48d00700b0 0x7f48d0096a30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:18.675 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.656+0000 7fa81affd640 1 -- 172.21.15.83:0/3822169017 >> v1:172.21.15.83:6800/1934165865 conn(0x7fa808047fd0 legacy=0x7fa80804a490 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v1:172.21.15.83:6800/1934165865 2024-09-14T03:40:18.675 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.656+0000 7f48c6ffd640 1 -- 172.21.15.83:0/2653415622 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f48d012cbd0 con 0x7f48d00709d0 2024-09-14T03:40:18.675 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.656+0000 7f48cffff640 1 --2- 172.21.15.83:0/2653415622 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f48d012f330 0x7f48d012b370 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-09-14T03:40:18.675 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.656+0000 7fb0d4ff9640 1 -- 172.21.15.83:0/463394540 <== osd.0 v1:172.21.15.83:6805/2271402019 1 ==== command_reply(tid 1: 0 ) ==== 8+0+27193 (unknown 0 0 1302949928) 0x7fb0d000de50 con 0x7fb0b804e740 2024-09-14T03:40:18.676 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.656+0000 7fa8037fe640 1 -- 172.21.15.83:0/3822169017 <== mon.1 v2:172.21.15.83:3301/0 6 ==== mon_get_version_reply(handle=1 version=20) ==== 24+0+0 (secure 0 0 0) 0x7fa81405fe60 con 0x7fa81c12bf30 2024-09-14T03:40:18.676 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.656+0000 7f85137fe640 1 -- 172.21.15.83:0/2656368403 <== mon.1 v2:172.21.15.83:3301/0 4 ==== mgrmap(e 5) ==== 55999+0+0 (secure 0 0 0) 0x7f8514004210 con 0x7f851c0949c0 2024-09-14T03:40:18.676 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.656+0000 7f48c6ffd640 1 --2- 172.21.15.83:0/2653415622 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f48d00709d0 0x7f48d0096f70 secure :-1 s=READY pgs=19 cs=0 l=1 rev1=1 crypto rx=0x7f48c000c440 tx=0x7f48c000c900 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:40:18.677 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.657+0000 7f851a59c640 1 -- 172.21.15.83:0/2656368403 >> v1:172.21.15.83:6800/1934165865 conn(0x7f84fc0481d0 legacy=0x7f84fc04a690 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v1:172.21.15.83:6800/1934165865 2024-09-14T03:40:18.677 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.657+0000 7f85137fe640 1 -- 172.21.15.83:0/2656368403 <== mon.1 v2:172.21.15.83:3301/0 5 ==== osd_map(20..20 src has 1..20) ==== 2812+0+0 (secure 0 0 0) 0x7f851403d0b0 con 0x7f851c0949c0 2024-09-14T03:40:18.677 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.657+0000 7f85137fe640 1 -- 172.21.15.83:0/2656368403 --> v1:172.21.15.83:6810/1553686396 -- command(tid 1: {"prefix": "get_command_descriptions"}) -- 0x7f84fc052b00 con 0x7f84fc04edf0 2024-09-14T03:40:18.677 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.657+0000 7f48ccff9640 1 -- 172.21.15.83:0/2653415622 <== mon.2 v2:172.21.15.83:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f48c0017070 con 0x7f48d00709d0 2024-09-14T03:40:18.677 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.657+0000 7f48ccff9640 1 -- 172.21.15.83:0/2653415622 <== mon.2 v2:172.21.15.83:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f48c00035f0 con 0x7f48d00709d0 2024-09-14T03:40:18.678 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.657+0000 7f48d611a640 1 -- 172.21.15.83:0/2653415622 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f48d012e860 con 0x7f48d00709d0 2024-09-14T03:40:18.678 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.657+0000 7f48d611a640 1 -- 172.21.15.83:0/2653415622 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7f48d012dfc0 con 0x7f48d00709d0 2024-09-14T03:40:18.678 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.658+0000 7f48ccff9640 1 -- 172.21.15.83:0/2653415622 <== mon.2 v2:172.21.15.83:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f48c0017070 con 0x7f48d00709d0 2024-09-14T03:40:18.678 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.659+0000 7f48d611a640 1 -- 172.21.15.83:0/2653415622 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_get_version(what=osdmap handle=1) -- 0x7f48d0070490 con 0x7f48d00709d0 2024-09-14T03:40:18.678 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.659+0000 7fa8037fe640 1 -- 172.21.15.83:0/3822169017 <== osd.3 v1:172.21.15.83:6809/2350261690 1 ==== command_reply(tid 1: 0 ) ==== 8+0+27193 (unknown 0 0 1302949928) 0x7fa814019e50 con 0x7fa80804e650 2024-09-14T03:40:18.678 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.659+0000 7f85137fe640 1 -- 172.21.15.83:0/2656368403 <== mon.1 v2:172.21.15.83:3301/0 6 ==== mon_get_version_reply(handle=1 version=20) ==== 24+0+0 (secure 0 0 0) 0x7f851403ba40 con 0x7f851c0949c0 2024-09-14T03:40:18.679 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.659+0000 7f85137fe640 1 -- 172.21.15.83:0/2656368403 <== osd.2 v1:172.21.15.83:6810/1553686396 1 ==== command_reply(tid 1: 0 ) ==== 8+0+27193 (unknown 0 0 1302949928) 0x7f84fc052b00 con 0x7f84fc04edf0 2024-09-14T03:40:18.679 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.660+0000 7f48ccff9640 1 -- 172.21.15.83:0/2653415622 <== mon.2 v2:172.21.15.83:3302/0 4 ==== mgrmap(e 5) ==== 55999+0+0 (secure 0 0 0) 0x7f48c00037a0 con 0x7f48d00709d0 2024-09-14T03:40:18.679 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.660+0000 7f48ccff9640 1 -- 172.21.15.83:0/2653415622 <== mon.2 v2:172.21.15.83:3302/0 5 ==== osd_map(20..20 src has 1..20) ==== 2812+0+0 (secure 0 0 0) 0x7f48c00052e0 con 0x7f48d00709d0 2024-09-14T03:40:18.679 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.660+0000 7f48cf7fe640 1 -- 172.21.15.83:0/2653415622 >> v1:172.21.15.83:6800/1934165865 conn(0x7f48b00483d0 legacy=0x7f48b004a890 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v1:172.21.15.83:6800/1934165865 2024-09-14T03:40:18.679 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.660+0000 7f48ccff9640 1 -- 172.21.15.83:0/2653415622 --> v1:172.21.15.83:6801/2493296278 -- command(tid 1: {"prefix": "get_command_descriptions"}) -- 0x7f48c000bf30 con 0x7f48b004ea50 2024-09-14T03:40:18.679 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.660+0000 7f48ccff9640 1 -- 172.21.15.83:0/2653415622 <== mon.2 v2:172.21.15.83:3302/0 6 ==== mon_get_version_reply(handle=1 version=20) ==== 24+0+0 (secure 0 0 0) 0x7f48c005cc00 con 0x7f48d00709d0 2024-09-14T03:40:18.679 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.662+0000 7f48ccff9640 1 -- 172.21.15.83:0/2653415622 <== osd.1 v1:172.21.15.83:6801/2493296278 1 ==== command_reply(tid 1: 0 ) ==== 8+0+27193 (unknown 0 0 1302949928) 0x7f48c000bf30 con 0x7f48b004ea50 2024-09-14T03:40:18.680 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.673+0000 7fb0de2ef640 1 -- 172.21.15.83:0/463394540 --> v1:172.21.15.83:6805/2271402019 -- command(tid 2: {"prefix": "flush_pg_stats"}) -- 0x7fb0d8071db0 con 0x7fb0b804e740 2024-09-14T03:40:18.680 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.674+0000 7fb0d4ff9640 1 -- 172.21.15.83:0/463394540 <== osd.0 v1:172.21.15.83:6805/2271402019 2 ==== command_reply(tid 2: 0 ) ==== 8+0+11 (unknown 0 0 1159357919) 0x7fb0d8071db0 con 0x7fb0b804e740 2024-09-14T03:40:18.681 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.674+0000 7fb0de2ef640 1 -- 172.21.15.83:0/463394540 >> v1:172.21.15.83:6805/2271402019 conn(0x7fb0b804e740 legacy=0x7fb0b8050b80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:18.681 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.674+0000 7fb0de2ef640 1 -- 172.21.15.83:0/463394540 >> v1:172.21.15.83:6800/1934165865 conn(0x7fb0b8048050 legacy=0x7fb0b804a510 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-09-14T03:40:18.681 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.674+0000 7fb0de2ef640 1 -- 172.21.15.83:0/463394540 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fb0d812a5c0 msgr2=0x7fb0d812a9d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:18.681 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.674+0000 7fb0de2ef640 1 --2- 172.21.15.83:0/463394540 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fb0d812a5c0 0x7fb0d812a9d0 secure :-1 s=READY pgs=15 cs=0 l=1 rev1=1 crypto rx=0x7fb0d000be00 tx=0x7fb0d000bef0 comp rx=0 tx=0).stop 2024-09-14T03:40:18.681 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.675+0000 7fb0dc88c640 1 -- 172.21.15.83:0/463394540 reap_dead start 2024-09-14T03:40:18.681 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.675+0000 7fb0de2ef640 1 -- 172.21.15.83:0/463394540 shutdown_connections 2024-09-14T03:40:18.681 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.675+0000 7fb0de2ef640 1 -- 172.21.15.83:0/463394540 >> 172.21.15.83:0/463394540 conn(0x7fb0d809d480 msgr2=0x7fb0d8070410 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:18.682 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.675+0000 7fb0de2ef640 1 -- 172.21.15.83:0/463394540 shutdown_connections 2024-09-14T03:40:18.682 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.675+0000 7fb0de2ef640 1 -- 172.21.15.83:0/463394540 wait complete. 2024-09-14T03:40:18.688 INFO:teuthology.orchestra.run.smithi083.stdout:38654705682 2024-09-14T03:40:18.688 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.0 2024-09-14T03:40:18.689 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.686+0000 7fa8217bb640 1 -- 172.21.15.83:0/3822169017 --> v1:172.21.15.83:6809/2350261690 -- command(tid 2: {"prefix": "flush_pg_stats"}) -- 0x7fa7fc002d50 con 0x7fa80804e650 2024-09-14T03:40:18.689 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.687+0000 7fa8037fe640 1 -- 172.21.15.83:0/3822169017 <== osd.3 v1:172.21.15.83:6809/2350261690 2 ==== command_reply(tid 2: 0 ) ==== 8+0+11 (unknown 0 0 4080920037) 0x7fa7fc002d50 con 0x7fa80804e650 2024-09-14T03:40:18.690 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.687+0000 7fa8217bb640 1 -- 172.21.15.83:0/3822169017 >> v1:172.21.15.83:6809/2350261690 conn(0x7fa80804e650 legacy=0x7fa808050a90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:18.690 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.687+0000 7fa8217bb640 1 -- 172.21.15.83:0/3822169017 >> v1:172.21.15.83:6800/1934165865 conn(0x7fa808047fd0 legacy=0x7fa80804a490 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-09-14T03:40:18.690 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.687+0000 7fa8217bb640 1 -- 172.21.15.83:0/3822169017 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fa81c12bf30 msgr2=0x7fa81c070190 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:18.690 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.687+0000 7fa8217bb640 1 --2- 172.21.15.83:0/3822169017 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fa81c12bf30 0x7fa81c070190 secure :-1 s=READY pgs=16 cs=0 l=1 rev1=1 crypto rx=0x7fa81400df70 tx=0x7fa81400b630 comp rx=0 tx=0).stop 2024-09-14T03:40:18.691 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.687+0000 7fa81b7fe640 1 -- 172.21.15.83:0/3822169017 reap_dead start 2024-09-14T03:40:18.691 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.687+0000 7fa8217bb640 1 -- 172.21.15.83:0/3822169017 shutdown_connections 2024-09-14T03:40:18.691 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.687+0000 7fa8217bb640 1 -- 172.21.15.83:0/3822169017 >> 172.21.15.83:0/3822169017 conn(0x7fa81c09e490 msgr2=0x7fa81c125140 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:18.691 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.688+0000 7fa8217bb640 1 -- 172.21.15.83:0/3822169017 shutdown_connections 2024-09-14T03:40:18.691 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.688+0000 7fa8217bb640 1 -- 172.21.15.83:0/3822169017 wait complete. 2024-09-14T03:40:18.692 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.688+0000 7f8521228640 1 -- 172.21.15.83:0/2656368403 --> v1:172.21.15.83:6810/1553686396 -- command(tid 2: {"prefix": "flush_pg_stats"}) -- 0x7f851c093150 con 0x7f84fc04edf0 2024-09-14T03:40:18.692 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.688+0000 7f85137fe640 1 -- 172.21.15.83:0/2656368403 <== osd.2 v1:172.21.15.83:6810/1553686396 2 ==== command_reply(tid 2: 0 ) ==== 8+0+11 (unknown 0 0 4080920037) 0x7f851c093150 con 0x7f84fc04edf0 2024-09-14T03:40:18.693 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.689+0000 7f8521228640 1 -- 172.21.15.83:0/2656368403 >> v1:172.21.15.83:6810/1553686396 conn(0x7f84fc04edf0 legacy=0x7f84fc051290 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:18.693 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.689+0000 7f8521228640 1 -- 172.21.15.83:0/2656368403 >> v1:172.21.15.83:6800/1934165865 conn(0x7f84fc0481d0 legacy=0x7f84fc04a690 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-09-14T03:40:18.693 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.689+0000 7f8521228640 1 -- 172.21.15.83:0/2656368403 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f851c0949c0 msgr2=0x7f851c1350e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:18.693 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.689+0000 7f8521228640 1 --2- 172.21.15.83:0/2656368403 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f851c0949c0 0x7f851c1350e0 secure :-1 s=READY pgs=17 cs=0 l=1 rev1=1 crypto rx=0x7f851400ea90 tx=0x7f851400ef50 comp rx=0 tx=0).stop 2024-09-14T03:40:18.693 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.689+0000 7f851b59e640 1 -- 172.21.15.83:0/2656368403 reap_dead start 2024-09-14T03:40:18.693 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.689+0000 7f8521228640 1 -- 172.21.15.83:0/2656368403 shutdown_connections 2024-09-14T03:40:18.693 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.689+0000 7f8521228640 1 -- 172.21.15.83:0/2656368403 >> 172.21.15.83:0/2656368403 conn(0x7f851c09d4f0 msgr2=0x7f851c0927f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:18.693 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.689+0000 7f8521228640 1 -- 172.21.15.83:0/2656368403 shutdown_connections 2024-09-14T03:40:18.693 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.689+0000 7f8521228640 1 -- 172.21.15.83:0/2656368403 wait complete. 2024-09-14T03:40:18.694 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.690+0000 7f48d611a640 1 -- 172.21.15.83:0/2653415622 --> v1:172.21.15.83:6801/2493296278 -- command(tid 2: {"prefix": "flush_pg_stats"}) -- 0x7f48d0099380 con 0x7f48b004ea50 2024-09-14T03:40:18.694 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.691+0000 7f48ccff9640 1 -- 172.21.15.83:0/2653415622 <== osd.1 v1:172.21.15.83:6801/2493296278 2 ==== command_reply(tid 2: 0 ) ==== 8+0+11 (unknown 0 0 4080920037) 0x7f48d0099380 con 0x7f48b004ea50 2024-09-14T03:40:18.695 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.692+0000 7f48d611a640 1 -- 172.21.15.83:0/2653415622 >> v1:172.21.15.83:6801/2493296278 conn(0x7f48b004ea50 legacy=0x7f48b0050e90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:18.695 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.692+0000 7f48d611a640 1 -- 172.21.15.83:0/2653415622 >> v1:172.21.15.83:6800/1934165865 conn(0x7f48b00483d0 legacy=0x7f48b004a890 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-09-14T03:40:18.695 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.692+0000 7f48d611a640 1 -- 172.21.15.83:0/2653415622 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f48d00709d0 msgr2=0x7f48d0096f70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:18.695 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.692+0000 7f48d611a640 1 --2- 172.21.15.83:0/2653415622 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f48d00709d0 0x7f48d0096f70 secure :-1 s=READY pgs=19 cs=0 l=1 rev1=1 crypto rx=0x7f48c000c440 tx=0x7f48c000c900 comp rx=0 tx=0).stop 2024-09-14T03:40:18.695 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.692+0000 7f48cffff640 1 -- 172.21.15.83:0/2653415622 reap_dead start 2024-09-14T03:40:18.696 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.692+0000 7f48d611a640 1 -- 172.21.15.83:0/2653415622 shutdown_connections 2024-09-14T03:40:18.696 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.692+0000 7f48d611a640 1 -- 172.21.15.83:0/2653415622 >> 172.21.15.83:0/2653415622 conn(0x7f48d009e0d0 msgr2=0x7f48d00989f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:18.696 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.692+0000 7f48d611a640 1 -- 172.21.15.83:0/2653415622 shutdown_connections 2024-09-14T03:40:18.696 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.693+0000 7f48d611a640 1 -- 172.21.15.83:0/2653415622 wait complete. 2024-09-14T03:40:18.709 INFO:teuthology.orchestra.run.smithi083.stdout:47244640274 2024-09-14T03:40:18.710 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.2 2024-09-14T03:40:18.710 INFO:teuthology.orchestra.run.smithi083.stdout:47244640274 2024-09-14T03:40:18.711 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.3 2024-09-14T03:40:18.714 INFO:teuthology.orchestra.run.smithi083.stdout:47244640274 2024-09-14T03:40:18.714 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.1 2024-09-14T03:40:18.824 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.820+0000 7efcecad4640 1 -- 172.21.15.83:0/3555694878 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7efce80715f0 msgr2=0x7efce80719d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:18.824 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.820+0000 7efcecad4640 1 --2- 172.21.15.83:0/3555694878 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7efce80715f0 0x7efce80719d0 secure :-1 s=READY pgs=18 cs=0 l=1 rev1=1 crypto rx=0x7efcd8005fd0 tx=0x7efcd802e440 comp rx=0 tx=0).stop 2024-09-14T03:40:18.824 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.821+0000 7efcecad4640 1 -- 172.21.15.83:0/3555694878 shutdown_connections 2024-09-14T03:40:18.824 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.821+0000 7efcecad4640 1 --2- 172.21.15.83:0/3555694878 >> v2:172.21.15.83:3300/0 conn(0x7efce8123260 0x7efce8134d40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:18.824 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.821+0000 7efcecad4640 1 --2- 172.21.15.83:0/3555694878 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7efce80715f0 0x7efce80719d0 unknown :-1 s=CLOSED pgs=18 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:18.824 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.821+0000 7efcecad4640 1 -- 172.21.15.83:0/3555694878 >> 172.21.15.83:0/3555694878 conn(0x7efce809d3a0 msgr2=0x7efce809d7b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:18.824 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.821+0000 7efcecad4640 1 -- 172.21.15.83:0/3555694878 shutdown_connections 2024-09-14T03:40:18.825 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.821+0000 7efcecad4640 1 -- 172.21.15.83:0/3555694878 wait complete. 2024-09-14T03:40:18.825 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.822+0000 7efcecad4640 1 Processor -- start 2024-09-14T03:40:18.825 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.822+0000 7efcecad4640 1 -- start start 2024-09-14T03:40:18.825 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.823+0000 7efcecad4640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7efce80715f0 0x7efce812ee00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:18.826 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.823+0000 7efce659c640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7efce80715f0 0x7efce812ee00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:18.826 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.823+0000 7efce659c640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7efce80715f0 0x7efce812ee00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3301/0 says I am v2:172.21.15.83:39244/0 (socket says 172.21.15.83:39244) 2024-09-14T03:40:18.827 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.823+0000 7efcecad4640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7efce8071f10 0x7efce812f340 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:18.827 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.823+0000 7efcecad4640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7efce8123260 0x7efce812f880 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:18.827 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.824+0000 7efce5d9b640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7efce8071f10 0x7efce812f340 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:18.827 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.824+0000 7efce5d9b640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7efce8071f10 0x7efce812f340 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3302/0 says I am v2:172.21.15.83:58378/0 (socket says 172.21.15.83:58378) 2024-09-14T03:40:18.827 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.824+0000 7efcecad4640 1 -- --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7efce806f880 con 0x7efce8123260 2024-09-14T03:40:18.827 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.824+0000 7efcecad4640 1 -- --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7efce806f700 con 0x7efce80715f0 2024-09-14T03:40:18.827 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.824+0000 7efcecad4640 1 -- --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7efce806fa00 con 0x7efce8071f10 2024-09-14T03:40:18.827 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.824+0000 7efce659c640 1 -- 172.21.15.83:0/1399307640 learned_addr learned my addr 172.21.15.83:0/1399307640 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:40:18.828 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.824+0000 7efce659c640 1 -- 172.21.15.83:0/1399307640 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7efce8071f10 msgr2=0x7efce812f340 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:18.828 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.824+0000 7efce6d9d640 1 --2- 172.21.15.83:0/1399307640 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7efce8123260 0x7efce812f880 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:18.828 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.824+0000 7efce659c640 1 --2- 172.21.15.83:0/1399307640 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7efce8071f10 0x7efce812f340 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:18.828 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.824+0000 7efce659c640 1 -- 172.21.15.83:0/1399307640 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7efce8123260 msgr2=0x7efce812f880 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:18.828 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.824+0000 7efce659c640 1 --2- 172.21.15.83:0/1399307640 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7efce8123260 0x7efce812f880 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:18.829 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.824+0000 7efce659c640 1 -- 172.21.15.83:0/1399307640 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7efce8126ee0 con 0x7efce80715f0 2024-09-14T03:40:18.829 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.825+0000 7efce5d9b640 1 --2- 172.21.15.83:0/1399307640 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7efce8071f10 0x7efce812f340 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-09-14T03:40:18.829 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.825+0000 7efce6d9d640 1 --2- 172.21.15.83:0/1399307640 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7efce8123260 0x7efce812f880 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-09-14T03:40:18.829 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.825+0000 7efce659c640 1 --2- 172.21.15.83:0/1399307640 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7efce80715f0 0x7efce812ee00 secure :-1 s=READY pgs=19 cs=0 l=1 rev1=1 crypto rx=0x7efcd8006010 tx=0x7efcd8010000 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:40:18.829 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.825+0000 7efcd6ffd640 1 -- 172.21.15.83:0/1399307640 <== mon.1 v2:172.21.15.83:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7efcd8041020 con 0x7efce80715f0 2024-09-14T03:40:18.829 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.825+0000 7efcd6ffd640 1 -- 172.21.15.83:0/1399307640 <== mon.1 v2:172.21.15.83:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7efcd800d2e0 con 0x7efce80715f0 2024-09-14T03:40:18.829 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.825+0000 7efcecad4640 1 -- 172.21.15.83:0/1399307640 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7efce81311d0 con 0x7efce80715f0 2024-09-14T03:40:18.829 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.825+0000 7efcd6ffd640 1 -- 172.21.15.83:0/1399307640 <== mon.1 v2:172.21.15.83:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7efcd8005280 con 0x7efce80715f0 2024-09-14T03:40:18.829 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.826+0000 7efcecad4640 1 -- 172.21.15.83:0/1399307640 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7efce812fdc0 con 0x7efce80715f0 2024-09-14T03:40:18.830 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.828+0000 7efcecad4640 1 -- 172.21.15.83:0/1399307640 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7efce80719d0 con 0x7efce80715f0 2024-09-14T03:40:18.831 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.828+0000 7efcd6ffd640 1 -- 172.21.15.83:0/1399307640 <== mon.1 v2:172.21.15.83:3301/0 4 ==== mgrmap(e 5) ==== 55999+0+0 (secure 0 0 0) 0x7efcd8005420 con 0x7efce80715f0 2024-09-14T03:40:18.832 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.829+0000 7efcd6ffd640 1 -- 172.21.15.83:0/1399307640 <== mon.1 v2:172.21.15.83:3301/0 5 ==== osd_map(20..20 src has 1..20) ==== 2812+0+0 (secure 0 0 0) 0x7efcd8083060 con 0x7efce80715f0 2024-09-14T03:40:18.832 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.829+0000 7efce5d9b640 1 -- 172.21.15.83:0/1399307640 >> v1:172.21.15.83:6800/1934165865 conn(0x7efcc8047fd0 legacy=0x7efcc804a490 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v1:172.21.15.83:6800/1934165865 2024-09-14T03:40:18.838 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.836+0000 7efcd6ffd640 1 -- 172.21.15.83:0/1399307640 <== mon.1 v2:172.21.15.83:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+152684 (secure 0 0 0) 0x7efcd8081e10 con 0x7efce80715f0 2024-09-14T03:40:18.919 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.916+0000 7fbc0f997640 1 -- 172.21.15.83:0/1974563177 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fbc08098d40 msgr2=0x7fbc08099120 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:18.919 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.916+0000 7fbc0f997640 1 --2- 172.21.15.83:0/1974563177 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fbc08098d40 0x7fbc08099120 secure :-1 s=READY pgs=28 cs=0 l=1 rev1=1 crypto rx=0x7fbc04009830 tx=0x7fbc0402a400 comp rx=0 tx=0).stop 2024-09-14T03:40:18.919 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.916+0000 7fbc0f997640 1 -- 172.21.15.83:0/1974563177 shutdown_connections 2024-09-14T03:40:18.919 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.916+0000 7fbc0f997640 1 --2- 172.21.15.83:0/1974563177 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fbc08099660 0x7fbc0812eab0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:18.919 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.916+0000 7fbc0f997640 1 --2- 172.21.15.83:0/1974563177 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fbc08098d40 0x7fbc08099120 unknown :-1 s=CLOSED pgs=28 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:18.919 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.916+0000 7fbc0f997640 1 -- 172.21.15.83:0/1974563177 >> 172.21.15.83:0/1974563177 conn(0x7fbc080a1790 msgr2=0x7fbc080a1ba0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:18.920 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.916+0000 7fbc0f997640 1 -- 172.21.15.83:0/1974563177 shutdown_connections 2024-09-14T03:40:18.920 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.916+0000 7fbc0f997640 1 -- 172.21.15.83:0/1974563177 wait complete. 2024-09-14T03:40:18.920 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.917+0000 7fbc0f997640 1 Processor -- start 2024-09-14T03:40:18.920 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.917+0000 7fbc0f997640 1 -- start start 2024-09-14T03:40:18.920 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.918+0000 7fbc0f997640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fbc08098d40 0x7fbc08071560 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:18.920 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.918+0000 7fbc0f997640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fbc08099660 0x7fbc08071aa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:18.921 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.918+0000 7fbc0f997640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fbc0812eff0 0x7fbc08071fe0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:18.921 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.918+0000 7fbc0f997640 1 -- --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7fbc081378c0 con 0x7fbc0812eff0 2024-09-14T03:40:18.921 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.918+0000 7fbc0f997640 1 -- --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7fbc08137740 con 0x7fbc08099660 2024-09-14T03:40:18.921 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.918+0000 7fbc0f997640 1 -- --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7fbc08137a40 con 0x7fbc08098d40 2024-09-14T03:40:18.921 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.918+0000 7fbc0d733640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fbc08098d40 0x7fbc08071560 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:18.921 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.918+0000 7fbc0d733640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fbc08098d40 0x7fbc08071560 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3302/0 says I am v2:172.21.15.83:58390/0 (socket says 172.21.15.83:58390) 2024-09-14T03:40:18.922 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.918+0000 7fbc0d733640 1 -- 172.21.15.83:0/2758909739 learned_addr learned my addr 172.21.15.83:0/2758909739 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:40:18.922 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.918+0000 7fbc0df34640 1 --2- 172.21.15.83:0/2758909739 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fbc0812eff0 0x7fbc08071fe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:18.922 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.919+0000 7fbc0cf32640 1 --2- 172.21.15.83:0/2758909739 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fbc08099660 0x7fbc08071aa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:18.922 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.920+0000 7fbc0cf32640 1 -- 172.21.15.83:0/2758909739 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fbc08098d40 msgr2=0x7fbc08071560 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:18.922 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.920+0000 7fbc0cf32640 1 --2- 172.21.15.83:0/2758909739 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fbc08098d40 0x7fbc08071560 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:18.922 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.920+0000 7fbc0cf32640 1 -- 172.21.15.83:0/2758909739 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fbc0812eff0 msgr2=0x7fbc08071fe0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:18.922 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.920+0000 7fbc0cf32640 1 --2- 172.21.15.83:0/2758909739 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fbc0812eff0 0x7fbc08071fe0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:18.922 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.920+0000 7fbc0cf32640 1 -- 172.21.15.83:0/2758909739 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fbc0812f6d0 con 0x7fbc08099660 2024-09-14T03:40:18.923 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.920+0000 7fbc0d733640 1 --2- 172.21.15.83:0/2758909739 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fbc08098d40 0x7fbc08071560 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:40:18.923 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.920+0000 7fbc0df34640 1 --2- 172.21.15.83:0/2758909739 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fbc0812eff0 0x7fbc08071fe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:40:18.923 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.920+0000 7fbc0cf32640 1 --2- 172.21.15.83:0/2758909739 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fbc08099660 0x7fbc08071aa0 secure :-1 s=READY pgs=21 cs=0 l=1 rev1=1 crypto rx=0x7fbbfc005fd0 tx=0x7fbbfc00dde0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:40:18.923 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.921+0000 7fbbf9ffb640 1 -- 172.21.15.83:0/2758909739 <== mon.1 v2:172.21.15.83:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fbbfc009c30 con 0x7fbc08099660 2024-09-14T03:40:18.923 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.921+0000 7fbbf9ffb640 1 -- 172.21.15.83:0/2758909739 <== mon.1 v2:172.21.15.83:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fbbfc004030 con 0x7fbc08099660 2024-09-14T03:40:18.923 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.921+0000 7fbc0f997640 1 -- 172.21.15.83:0/2758909739 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fbc08073d90 con 0x7fbc08099660 2024-09-14T03:40:18.924 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.921+0000 7fbbf9ffb640 1 -- 172.21.15.83:0/2758909739 <== mon.1 v2:172.21.15.83:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fbbfc004440 con 0x7fbc08099660 2024-09-14T03:40:18.924 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.922+0000 7f7c1e673640 1 -- 172.21.15.83:0/1880555269 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f7c18099150 msgr2=0x7f7c180995e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:18.925 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.922+0000 7f7c1e673640 1 --2- 172.21.15.83:0/1880555269 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f7c18099150 0x7f7c180995e0 secure :-1 s=READY pgs=20 cs=0 l=1 rev1=1 crypto rx=0x7f7c10005fd0 tx=0x7f7c1002a710 comp rx=0 tx=0).stop 2024-09-14T03:40:18.925 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.922+0000 7fbc0f997640 1 -- 172.21.15.83:0/2758909739 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7fbc08136270 con 0x7fbc08099660 2024-09-14T03:40:18.925 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.923+0000 7f7c1e673640 1 -- 172.21.15.83:0/1880555269 shutdown_connections 2024-09-14T03:40:18.925 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.923+0000 7f7c1e673640 1 --2- 172.21.15.83:0/1880555269 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f7c18099150 0x7f7c180995e0 unknown :-1 s=CLOSED pgs=20 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:18.925 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.923+0000 7f7c1e673640 1 --2- 172.21.15.83:0/1880555269 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f7c18098d70 0x7f7c18098020 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:18.925 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.923+0000 7f7c1e673640 1 --2- 172.21.15.83:0/1880555269 >> v2:172.21.15.83:3300/0 conn(0x7f7c1812f0e0 0x7f7c1812f4c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:18.925 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.923+0000 7f7c1e673640 1 -- 172.21.15.83:0/1880555269 >> 172.21.15.83:0/1880555269 conn(0x7f7c180a1790 msgr2=0x7f7c180a1ba0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:18.926 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.923+0000 7f7c1e673640 1 -- 172.21.15.83:0/1880555269 shutdown_connections 2024-09-14T03:40:18.926 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.923+0000 7fbbf9ffb640 1 -- 172.21.15.83:0/2758909739 <== mon.1 v2:172.21.15.83:3301/0 4 ==== mgrmap(e 5) ==== 55999+0+0 (secure 0 0 0) 0x7fbbfc0035f0 con 0x7fbc08099660 2024-09-14T03:40:18.926 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.923+0000 7fbbf9ffb640 1 -- 172.21.15.83:0/2758909739 <== mon.1 v2:172.21.15.83:3301/0 5 ==== osd_map(20..20 src has 1..20) ==== 2812+0+0 (secure 0 0 0) 0x7fbbfc05dba0 con 0x7fbc08099660 2024-09-14T03:40:18.927 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.923+0000 7fbc0d733640 1 -- 172.21.15.83:0/2758909739 >> v1:172.21.15.83:6800/1934165865 conn(0x7fbbec047fd0 legacy=0x7fbbec04a490 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v1:172.21.15.83:6800/1934165865 2024-09-14T03:40:18.927 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.924+0000 7fbc0f997640 1 -- 172.21.15.83:0/2758909739 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fbc0809d060 con 0x7fbc08099660 2024-09-14T03:40:18.928 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.923+0000 7f7c1e673640 1 -- 172.21.15.83:0/1880555269 wait complete. 2024-09-14T03:40:18.928 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.924+0000 7f7c1e673640 1 Processor -- start 2024-09-14T03:40:18.928 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.924+0000 7f7c1e673640 1 -- start start 2024-09-14T03:40:18.928 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.925+0000 7f7c1e673640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f7c18098d70 0x7f7c1806f910 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:18.928 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.925+0000 7f7c1e673640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f7c18099150 0x7f7c1806fe50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:18.928 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.925+0000 7f7c1e673640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f7c1812f0e0 0x7f7c18070390 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:18.928 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.925+0000 7f7c1e673640 1 -- --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7f7c1813abf0 con 0x7f7c1812f0e0 2024-09-14T03:40:18.928 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.925+0000 7f7c1e673640 1 -- --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7f7c1813aa70 con 0x7f7c18099150 2024-09-14T03:40:18.928 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.925+0000 7f7c1e673640 1 -- --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7f7c1813ad70 con 0x7f7c18098d70 2024-09-14T03:40:18.929 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.925+0000 7f7c177fe640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f7c18099150 0x7f7c1806fe50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:18.929 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.925+0000 7f7c177fe640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f7c18099150 0x7f7c1806fe50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3301/0 says I am v2:172.21.15.83:39286/0 (socket says 172.21.15.83:39286) 2024-09-14T03:40:18.929 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.925+0000 7f7c177fe640 1 -- 172.21.15.83:0/297118073 learned_addr learned my addr 172.21.15.83:0/297118073 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:40:18.929 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.925+0000 7f7c17fff640 1 --2- 172.21.15.83:0/297118073 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f7c18098d70 0x7f7c1806f910 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:18.929 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.926+0000 7f7c1cc10640 1 --2- 172.21.15.83:0/297118073 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f7c1812f0e0 0x7f7c18070390 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:18.930 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.926+0000 7f7c17fff640 1 -- 172.21.15.83:0/297118073 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f7c18099150 msgr2=0x7f7c1806fe50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:18.930 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.927+0000 7f7c17fff640 1 --2- 172.21.15.83:0/297118073 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f7c18099150 0x7f7c1806fe50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:18.930 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.927+0000 7f7c17fff640 1 -- 172.21.15.83:0/297118073 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f7c1812f0e0 msgr2=0x7f7c18070390 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:18.930 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.927+0000 7f7c17fff640 1 --2- 172.21.15.83:0/297118073 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f7c1812f0e0 0x7f7c18070390 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:18.930 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.927+0000 7f7c17fff640 1 -- 172.21.15.83:0/297118073 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f7c18072ee0 con 0x7f7c18098d70 2024-09-14T03:40:18.930 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.927+0000 7f7c177fe640 1 --2- 172.21.15.83:0/297118073 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f7c18099150 0x7f7c1806fe50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-09-14T03:40:18.930 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.927+0000 7f7c1cc10640 1 --2- 172.21.15.83:0/297118073 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f7c1812f0e0 0x7f7c18070390 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:40:18.930 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.927+0000 7f7c17fff640 1 --2- 172.21.15.83:0/297118073 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f7c18098d70 0x7f7c1806f910 secure :-1 s=READY pgs=20 cs=0 l=1 rev1=1 crypto rx=0x7f7c0800d6d0 tx=0x7f7c0800db90 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:40:18.932 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.929+0000 7f7c14ff9640 1 -- 172.21.15.83:0/297118073 <== mon.2 v2:172.21.15.83:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7c08017070 con 0x7f7c18098d70 2024-09-14T03:40:18.932 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.929+0000 7f7c14ff9640 1 -- 172.21.15.83:0/297118073 <== mon.2 v2:172.21.15.83:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f7c08005310 con 0x7f7c18098d70 2024-09-14T03:40:18.932 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.929+0000 7f7c14ff9640 1 -- 172.21.15.83:0/297118073 <== mon.2 v2:172.21.15.83:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7c08009820 con 0x7f7c18098d70 2024-09-14T03:40:18.932 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.929+0000 7f7c1e673640 1 -- 172.21.15.83:0/297118073 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f7c18072330 con 0x7f7c18098d70 2024-09-14T03:40:18.932 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.929+0000 7f7c1e673640 1 -- 172.21.15.83:0/297118073 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7f7c18073380 con 0x7f7c18098d70 2024-09-14T03:40:18.933 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.931+0000 7fbbf9ffb640 1 -- 172.21.15.83:0/2758909739 <== mon.1 v2:172.21.15.83:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+152684 (secure 0 0 0) 0x7fbc0809d060 con 0x7fbc08099660 2024-09-14T03:40:18.933 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.931+0000 7f7c14ff9640 1 -- 172.21.15.83:0/297118073 <== mon.2 v2:172.21.15.83:3302/0 4 ==== mgrmap(e 5) ==== 55999+0+0 (secure 0 0 0) 0x7f7c080099c0 con 0x7f7c18098d70 2024-09-14T03:40:18.934 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.931+0000 7f7c1e673640 1 -- 172.21.15.83:0/297118073 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f7bfc005290 con 0x7f7c18098d70 2024-09-14T03:40:18.934 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.931+0000 7f7c177fe640 1 -- 172.21.15.83:0/297118073 >> v1:172.21.15.83:6800/1934165865 conn(0x7f7bf8048050 legacy=0x7f7bf804a510 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v1:172.21.15.83:6800/1934165865 2024-09-14T03:40:18.934 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.931+0000 7f9f54bdc640 1 -- 172.21.15.83:0/256141093 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f9f50095510 msgr2=0x7f9f50082fd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:18.934 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.931+0000 7f9f54bdc640 1 --2- 172.21.15.83:0/256141093 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f9f50095510 0x7f9f50082fd0 secure :-1 s=READY pgs=30 cs=0 l=1 rev1=1 crypto rx=0x7f9f38007350 tx=0x7f9f3802b110 comp rx=0 tx=0).stop 2024-09-14T03:40:18.934 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.931+0000 7f7c14ff9640 1 -- 172.21.15.83:0/297118073 <== mon.2 v2:172.21.15.83:3302/0 5 ==== osd_map(20..20 src has 1..20) ==== 2812+0+0 (secure 0 0 0) 0x7f7c0803cbd0 con 0x7f7c18098d70 2024-09-14T03:40:18.934 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.931+0000 7f9f54bdc640 1 -- 172.21.15.83:0/256141093 shutdown_connections 2024-09-14T03:40:18.935 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.931+0000 7f9f54bdc640 1 --2- 172.21.15.83:0/256141093 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f9f50083510 0x7f9f50085950 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:18.935 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.931+0000 7f9f54bdc640 1 --2- 172.21.15.83:0/256141093 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f9f50095510 0x7f9f50082fd0 unknown :-1 s=CLOSED pgs=30 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:18.935 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.931+0000 7f9f54bdc640 1 -- 172.21.15.83:0/256141093 >> 172.21.15.83:0/256141093 conn(0x7f9f5009d4f0 msgr2=0x7f9f5009d900 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:18.935 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.933+0000 7f9f54bdc640 1 -- 172.21.15.83:0/256141093 shutdown_connections 2024-09-14T03:40:18.935 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.933+0000 7f9f54bdc640 1 -- 172.21.15.83:0/256141093 wait complete. 2024-09-14T03:40:18.936 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.934+0000 7f9f54bdc640 1 Processor -- start 2024-09-14T03:40:18.936 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.934+0000 7f9f54bdc640 1 -- start start 2024-09-14T03:40:18.937 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.935+0000 7f9f54bdc640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f9f50083510 0x7f9f50097ad0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:18.937 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.935+0000 7f9f54bdc640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f9f50094a20 0x7f9f50098010 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:18.939 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.935+0000 7f9f54bdc640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f9f50095510 0x7f9f50098550 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:18.939 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.935+0000 7f9f54bdc640 1 -- --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7f9f501447d0 con 0x7f9f50083510 2024-09-14T03:40:18.939 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.935+0000 7f9f54bdc640 1 -- --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7f9f50144650 con 0x7f9f50095510 2024-09-14T03:40:18.939 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.935+0000 7f9f54bdc640 1 -- --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7f9f50144950 con 0x7f9f50094a20 2024-09-14T03:40:18.939 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.935+0000 7f9f4ed9d640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f9f50095510 0x7f9f50098550 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:18.939 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.935+0000 7f9f4ed9d640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f9f50095510 0x7f9f50098550 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3301/0 says I am v2:172.21.15.83:39288/0 (socket says 172.21.15.83:39288) 2024-09-14T03:40:18.940 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.935+0000 7f9f4ed9d640 1 -- 172.21.15.83:0/794352876 learned_addr learned my addr 172.21.15.83:0/794352876 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:40:18.940 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.935+0000 7f9f47fff640 1 --2- 172.21.15.83:0/794352876 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f9f50094a20 0x7f9f50098010 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:18.940 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.936+0000 7f9f4ed9d640 1 -- 172.21.15.83:0/794352876 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f9f50094a20 msgr2=0x7f9f50098010 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:18.940 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.936+0000 7f9f4ed9d640 1 --2- 172.21.15.83:0/794352876 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f9f50094a20 0x7f9f50098010 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:18.940 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.936+0000 7f9f4ed9d640 1 -- 172.21.15.83:0/794352876 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f9f50083510 msgr2=0x7f9f50097ad0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:18.940 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.935+0000 7f9f4e59c640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f9f50083510 0x7f9f50097ad0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:18.940 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.936+0000 7f9f4ed9d640 1 --2- 172.21.15.83:0/794352876 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f9f50083510 0x7f9f50097ad0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:18.940 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.936+0000 7f9f4ed9d640 1 -- 172.21.15.83:0/794352876 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f9f50083310 con 0x7f9f50095510 2024-09-14T03:40:18.940 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.936+0000 7f9f4e59c640 1 --2- 172.21.15.83:0/794352876 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f9f50083510 0x7f9f50097ad0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-09-14T03:40:18.940 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.936+0000 7f9f4ed9d640 1 --2- 172.21.15.83:0/794352876 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f9f50095510 0x7f9f50098550 secure :-1 s=READY pgs=22 cs=0 l=1 rev1=1 crypto rx=0x7f9f4800e960 tx=0x7f9f4800ee20 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:40:18.940 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.937+0000 7f9f46ffd640 1 -- 172.21.15.83:0/794352876 <== mon.1 v2:172.21.15.83:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9f48004d20 con 0x7f9f50095510 2024-09-14T03:40:18.940 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.937+0000 7f9f46ffd640 1 -- 172.21.15.83:0/794352876 <== mon.1 v2:172.21.15.83:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f9f4800cb20 con 0x7f9f50095510 2024-09-14T03:40:18.940 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.937+0000 7f9f46ffd640 1 -- 172.21.15.83:0/794352876 <== mon.1 v2:172.21.15.83:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9f48005020 con 0x7f9f50095510 2024-09-14T03:40:18.941 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.937+0000 7f9f54bdc640 1 -- 172.21.15.83:0/794352876 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f9f50073910 con 0x7f9f50095510 2024-09-14T03:40:18.941 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.937+0000 7f9f54bdc640 1 -- 172.21.15.83:0/794352876 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7f9f5008dd10 con 0x7f9f50095510 2024-09-14T03:40:18.942 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.938+0000 7f9f46ffd640 1 -- 172.21.15.83:0/794352876 <== mon.1 v2:172.21.15.83:3301/0 4 ==== mgrmap(e 5) ==== 55999+0+0 (secure 0 0 0) 0x7f9f480051c0 con 0x7f9f50095510 2024-09-14T03:40:18.942 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.939+0000 7f9f46ffd640 1 -- 172.21.15.83:0/794352876 <== mon.1 v2:172.21.15.83:3301/0 5 ==== osd_map(20..20 src has 1..20) ==== 2812+0+0 (secure 0 0 0) 0x7f9f4805e210 con 0x7f9f50095510 2024-09-14T03:40:18.942 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.939+0000 7f9f4e59c640 1 -- 172.21.15.83:0/794352876 >> v1:172.21.15.83:6800/1934165865 conn(0x7f9f300480a0 legacy=0x7f9f3004a560 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v1:172.21.15.83:6800/1934165865 2024-09-14T03:40:18.942 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.939+0000 7f9f54bdc640 1 -- 172.21.15.83:0/794352876 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f9f50094e00 con 0x7f9f50095510 2024-09-14T03:40:18.942 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.939+0000 7f7c14ff9640 1 -- 172.21.15.83:0/297118073 <== mon.2 v2:172.21.15.83:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+152684 (secure 0 0 0) 0x7f7c0803e670 con 0x7f7c18098d70 2024-09-14T03:40:18.949 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:18.946+0000 7f9f46ffd640 1 -- 172.21.15.83:0/794352876 <== mon.1 v2:172.21.15.83:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+152684 (secure 0 0 0) 0x7f9f48016030 con 0x7f9f50095510 2024-09-14T03:40:19.033 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:19.030+0000 7efce5d9b640 1 -- 172.21.15.83:0/1399307640 >> v1:172.21.15.83:6800/1934165865 conn(0x7efcc8047fd0 legacy=0x7efcc804a490 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v1:172.21.15.83:6800/1934165865 2024-09-14T03:40:19.126 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:19.124+0000 7fbc0d733640 1 -- 172.21.15.83:0/2758909739 >> v1:172.21.15.83:6800/1934165865 conn(0x7fbbec047fd0 legacy=0x7fbbec04a490 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v1:172.21.15.83:6800/1934165865 2024-09-14T03:40:19.135 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:19.132+0000 7f7c177fe640 1 -- 172.21.15.83:0/297118073 >> v1:172.21.15.83:6800/1934165865 conn(0x7f7bf8048050 legacy=0x7f7bf804a510 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v1:172.21.15.83:6800/1934165865 2024-09-14T03:40:19.143 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:19.141+0000 7f9f4e59c640 1 -- 172.21.15.83:0/794352876 >> v1:172.21.15.83:6800/1934165865 conn(0x7f9f300480a0 legacy=0x7f9f3004a560 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v1:172.21.15.83:6800/1934165865 2024-09-14T03:40:19.158 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:19.156+0000 7efcecad4640 1 -- 172.21.15.83:0/1399307640 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 0} v 0) -- 0x7efce8129810 con 0x7efce80715f0 2024-09-14T03:40:19.159 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:19.157+0000 7efcd6ffd640 1 -- 172.21.15.83:0/1399307640 <== mon.1 v2:172.21.15.83:3301/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 0}]=0 v0) ==== 74+0+12 (secure 0 0 0) 0x7efcd805ca80 con 0x7efce80715f0 2024-09-14T03:40:19.159 INFO:teuthology.orchestra.run.smithi083.stdout:38654705681 2024-09-14T03:40:19.162 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:19.159+0000 7efcecad4640 1 -- 172.21.15.83:0/1399307640 >> v1:172.21.15.83:6800/1934165865 conn(0x7efcc8047fd0 legacy=0x7efcc804a490 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-09-14T03:40:19.162 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:19.160+0000 7efcecad4640 1 -- 172.21.15.83:0/1399307640 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7efce80715f0 msgr2=0x7efce812ee00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:19.162 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:19.160+0000 7efcecad4640 1 --2- 172.21.15.83:0/1399307640 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7efce80715f0 0x7efce812ee00 secure :-1 s=READY pgs=19 cs=0 l=1 rev1=1 crypto rx=0x7efcd8006010 tx=0x7efcd8010000 comp rx=0 tx=0).stop 2024-09-14T03:40:19.162 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:19.160+0000 7efcecad4640 1 -- 172.21.15.83:0/1399307640 shutdown_connections 2024-09-14T03:40:19.162 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:19.160+0000 7efcecad4640 1 --2- 172.21.15.83:0/1399307640 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7efce8123260 0x7efce812f880 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:19.162 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:19.160+0000 7efcecad4640 1 --2- 172.21.15.83:0/1399307640 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7efce8071f10 0x7efce812f340 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:19.163 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:19.160+0000 7efcecad4640 1 --2- 172.21.15.83:0/1399307640 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7efce80715f0 0x7efce812ee00 unknown :-1 s=CLOSED pgs=19 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:19.163 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:19.160+0000 7efcecad4640 1 -- 172.21.15.83:0/1399307640 >> 172.21.15.83:0/1399307640 conn(0x7efce809d3a0 msgr2=0x7efce809b680 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:19.163 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:19.160+0000 7efcecad4640 1 -- 172.21.15.83:0/1399307640 shutdown_connections 2024-09-14T03:40:19.163 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:19.160+0000 7efcecad4640 1 -- 172.21.15.83:0/1399307640 wait complete. 2024-09-14T03:40:19.179 INFO:tasks.ceph.ceph_manager.ceph:need seq 38654705682 got 38654705681 for osd.0 2024-09-14T03:40:19.228 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:19.226+0000 7f9f54bdc640 1 -- 172.21.15.83:0/794352876 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 1} v 0) -- 0x7f9f50074a00 con 0x7f9f50095510 2024-09-14T03:40:19.229 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:19.226+0000 7f9f46ffd640 1 -- 172.21.15.83:0/794352876 <== mon.1 v2:172.21.15.83:3301/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 1}]=0 v0) ==== 74+0+12 (secure 0 0 0) 0x7f9f4803d460 con 0x7f9f50095510 2024-09-14T03:40:19.229 INFO:teuthology.orchestra.run.smithi083.stdout:47244640273 2024-09-14T03:40:19.231 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:19.228+0000 7f9f54bdc640 1 -- 172.21.15.83:0/794352876 >> v1:172.21.15.83:6800/1934165865 conn(0x7f9f300480a0 legacy=0x7f9f3004a560 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-09-14T03:40:19.231 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:19.228+0000 7f9f54bdc640 1 -- 172.21.15.83:0/794352876 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f9f50095510 msgr2=0x7f9f50098550 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:19.231 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:19.228+0000 7f9f54bdc640 1 --2- 172.21.15.83:0/794352876 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f9f50095510 0x7f9f50098550 secure :-1 s=READY pgs=22 cs=0 l=1 rev1=1 crypto rx=0x7f9f4800e960 tx=0x7f9f4800ee20 comp rx=0 tx=0).stop 2024-09-14T03:40:19.231 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:19.229+0000 7f9f54bdc640 1 -- 172.21.15.83:0/794352876 shutdown_connections 2024-09-14T03:40:19.231 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:19.229+0000 7f9f54bdc640 1 --2- 172.21.15.83:0/794352876 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f9f50095510 0x7f9f50098550 unknown :-1 s=CLOSED pgs=22 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:19.232 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:19.229+0000 7f9f54bdc640 1 --2- 172.21.15.83:0/794352876 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f9f50094a20 0x7f9f50098010 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:19.232 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:19.229+0000 7f9f54bdc640 1 --2- 172.21.15.83:0/794352876 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f9f50083510 0x7f9f50097ad0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:19.232 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:19.229+0000 7f9f54bdc640 1 -- 172.21.15.83:0/794352876 >> 172.21.15.83:0/794352876 conn(0x7f9f5009d4f0 msgr2=0x7f9f500882d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:19.232 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:19.229+0000 7f9f54bdc640 1 -- 172.21.15.83:0/794352876 shutdown_connections 2024-09-14T03:40:19.232 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:19.229+0000 7f9f54bdc640 1 -- 172.21.15.83:0/794352876 wait complete. 2024-09-14T03:40:19.234 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:19.232+0000 7f7c1e673640 1 -- 172.21.15.83:0/297118073 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 2} v 0) -- 0x7f7bfc005850 con 0x7f7c18098d70 2024-09-14T03:40:19.235 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:19.233+0000 7f7c14ff9640 1 -- 172.21.15.83:0/297118073 <== mon.2 v2:172.21.15.83:3302/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 2}]=0 v0) ==== 74+0+12 (secure 0 0 0) 0x7f7c0803a880 con 0x7f7c18098d70 2024-09-14T03:40:19.235 INFO:teuthology.orchestra.run.smithi083.stdout:47244640273 2024-09-14T03:40:19.238 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:19.236+0000 7f7c1e673640 1 -- 172.21.15.83:0/297118073 >> v1:172.21.15.83:6800/1934165865 conn(0x7f7bf8048050 legacy=0x7f7bf804a510 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-09-14T03:40:19.238 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:19.236+0000 7f7c1e673640 1 -- 172.21.15.83:0/297118073 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f7c18098d70 msgr2=0x7f7c1806f910 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:19.239 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:19.236+0000 7f7c1e673640 1 --2- 172.21.15.83:0/297118073 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f7c18098d70 0x7f7c1806f910 secure :-1 s=READY pgs=20 cs=0 l=1 rev1=1 crypto rx=0x7f7c0800d6d0 tx=0x7f7c0800db90 comp rx=0 tx=0).stop 2024-09-14T03:40:19.239 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:19.236+0000 7f7c1e673640 1 -- 172.21.15.83:0/297118073 shutdown_connections 2024-09-14T03:40:19.239 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:19.236+0000 7f7c1e673640 1 --2- 172.21.15.83:0/297118073 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f7c1812f0e0 0x7f7c18070390 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:19.239 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:19.236+0000 7f7c1e673640 1 --2- 172.21.15.83:0/297118073 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f7c18099150 0x7f7c1806fe50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:19.239 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:19.236+0000 7f7c1e673640 1 --2- 172.21.15.83:0/297118073 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f7c18098d70 0x7f7c1806f910 unknown :-1 s=CLOSED pgs=20 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:19.239 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:19.236+0000 7f7c1e673640 1 -- 172.21.15.83:0/297118073 >> 172.21.15.83:0/297118073 conn(0x7f7c180a1790 msgr2=0x7f7c18129670 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:19.239 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:19.236+0000 7f7c1e673640 1 -- 172.21.15.83:0/297118073 shutdown_connections 2024-09-14T03:40:19.239 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:19.236+0000 7f7c1e673640 1 -- 172.21.15.83:0/297118073 wait complete. 2024-09-14T03:40:19.246 INFO:tasks.ceph.ceph_manager.ceph:need seq 47244640274 got 47244640273 for osd.1 2024-09-14T03:40:19.247 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:19.244+0000 7fbc0f997640 1 -- 172.21.15.83:0/2758909739 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 3} v 0) -- 0x7fbc08099120 con 0x7fbc08099660 2024-09-14T03:40:19.247 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:19.245+0000 7fbbf9ffb640 1 -- 172.21.15.83:0/2758909739 <== mon.1 v2:172.21.15.83:3301/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 3}]=0 v0) ==== 74+0+12 (secure 0 0 0) 0x7fbbfc017020 con 0x7fbc08099660 2024-09-14T03:40:19.247 INFO:teuthology.orchestra.run.smithi083.stdout:47244640273 2024-09-14T03:40:19.251 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:19.248+0000 7fbc0f997640 1 -- 172.21.15.83:0/2758909739 >> v1:172.21.15.83:6800/1934165865 conn(0x7fbbec047fd0 legacy=0x7fbbec04a490 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-09-14T03:40:19.251 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:19.248+0000 7fbc0f997640 1 -- 172.21.15.83:0/2758909739 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fbc08099660 msgr2=0x7fbc08071aa0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:19.251 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:19.248+0000 7fbc0f997640 1 --2- 172.21.15.83:0/2758909739 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fbc08099660 0x7fbc08071aa0 secure :-1 s=READY pgs=21 cs=0 l=1 rev1=1 crypto rx=0x7fbbfc005fd0 tx=0x7fbbfc00dde0 comp rx=0 tx=0).stop 2024-09-14T03:40:19.251 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:19.248+0000 7fbc0f997640 1 -- 172.21.15.83:0/2758909739 shutdown_connections 2024-09-14T03:40:19.251 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:19.248+0000 7fbc0f997640 1 --2- 172.21.15.83:0/2758909739 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fbc0812eff0 0x7fbc08071fe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:19.251 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:19.248+0000 7fbc0f997640 1 --2- 172.21.15.83:0/2758909739 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fbc08099660 0x7fbc08071aa0 unknown :-1 s=CLOSED pgs=21 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:19.251 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:19.248+0000 7fbc0f997640 1 --2- 172.21.15.83:0/2758909739 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fbc08098d40 0x7fbc08071560 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:19.251 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:19.248+0000 7fbc0f997640 1 -- 172.21.15.83:0/2758909739 >> 172.21.15.83:0/2758909739 conn(0x7fbc080a1790 msgr2=0x7fbc081366b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:19.251 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:19.248+0000 7fbc0f997640 1 -- 172.21.15.83:0/2758909739 shutdown_connections 2024-09-14T03:40:19.251 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:19.248+0000 7fbc0f997640 1 -- 172.21.15.83:0/2758909739 wait complete. 2024-09-14T03:40:19.252 INFO:tasks.ceph.ceph_manager.ceph:need seq 47244640274 got 47244640273 for osd.2 2024-09-14T03:40:19.265 INFO:tasks.ceph.ceph_manager.ceph:need seq 47244640274 got 47244640273 for osd.3 2024-09-14T03:40:20.179 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.0 2024-09-14T03:40:20.247 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.1 2024-09-14T03:40:20.253 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.2 2024-09-14T03:40:20.266 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.3 2024-09-14T03:40:20.282 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.279+0000 7f6ba6839640 1 -- 172.21.15.83:0/1687786931 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f6ba012bdb0 msgr2=0x7f6ba0094e60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:20.283 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.279+0000 7f6ba6839640 1 --2- 172.21.15.83:0/1687786931 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f6ba012bdb0 0x7f6ba0094e60 secure :-1 s=READY pgs=24 cs=0 l=1 rev1=1 crypto rx=0x7f6b94009830 tx=0x7f6b9402a3f0 comp rx=0 tx=0).stop 2024-09-14T03:40:20.283 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.279+0000 7f6ba6839640 1 -- 172.21.15.83:0/1687786931 shutdown_connections 2024-09-14T03:40:20.283 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.279+0000 7f6ba6839640 1 --2- 172.21.15.83:0/1687786931 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f6ba00953a0 0x7f6ba0095810 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:20.283 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.279+0000 7f6ba6839640 1 --2- 172.21.15.83:0/1687786931 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f6ba012bdb0 0x7f6ba0094e60 unknown :-1 s=CLOSED pgs=24 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:20.283 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.279+0000 7f6ba6839640 1 --2- 172.21.15.83:0/1687786931 >> v2:172.21.15.83:3300/0 conn(0x7f6ba012b490 0x7f6ba012b870 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:20.283 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.279+0000 7f6ba6839640 1 -- 172.21.15.83:0/1687786931 >> 172.21.15.83:0/1687786931 conn(0x7f6ba009afd0 msgr2=0x7f6ba009b3e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:20.283 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.279+0000 7f6ba6839640 1 -- 172.21.15.83:0/1687786931 shutdown_connections 2024-09-14T03:40:20.283 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.279+0000 7f6ba6839640 1 -- 172.21.15.83:0/1687786931 wait complete. 2024-09-14T03:40:20.283 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.280+0000 7f6ba6839640 1 Processor -- start 2024-09-14T03:40:20.283 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.280+0000 7f6ba6839640 1 -- start start 2024-09-14T03:40:20.284 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.281+0000 7f6ba6839640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f6ba00953a0 0x7f6ba01c0a20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:20.284 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.281+0000 7f6ba6839640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f6ba012b490 0x7f6ba01c0f60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:20.284 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.281+0000 7f6ba6839640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f6ba012bdb0 0x7f6ba0130370 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:20.285 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.281+0000 7f6ba6839640 1 -- --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7f6ba0132470 con 0x7f6ba012b490 2024-09-14T03:40:20.285 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.281+0000 7f6ba6839640 1 -- --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7f6ba0094b80 con 0x7f6ba012bdb0 2024-09-14T03:40:20.285 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.281+0000 7f6ba6839640 1 -- --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7f6ba01325f0 con 0x7f6ba00953a0 2024-09-14T03:40:20.285 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.281+0000 7f6b9ffff640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f6ba00953a0 0x7f6ba01c0a20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:20.285 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.281+0000 7f6b9f7fe640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f6ba012b490 0x7f6ba01c0f60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:20.285 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.281+0000 7f6b9ffff640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f6ba00953a0 0x7f6ba01c0a20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3302/0 says I am v2:172.21.15.83:58516/0 (socket says 172.21.15.83:58516) 2024-09-14T03:40:20.285 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.281+0000 7f6b9ffff640 1 -- 172.21.15.83:0/4087154445 learned_addr learned my addr 172.21.15.83:0/4087154445 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:40:20.285 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.281+0000 7f6ba4dd6640 1 --2- 172.21.15.83:0/4087154445 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f6ba012bdb0 0x7f6ba0130370 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:20.285 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.281+0000 7f6b9ffff640 1 -- 172.21.15.83:0/4087154445 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f6ba012bdb0 msgr2=0x7f6ba0130370 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:20.285 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.281+0000 7f6b9ffff640 1 --2- 172.21.15.83:0/4087154445 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f6ba012bdb0 0x7f6ba0130370 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:20.285 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.281+0000 7f6b9ffff640 1 -- 172.21.15.83:0/4087154445 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f6ba012b490 msgr2=0x7f6ba01c0f60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:20.285 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.281+0000 7f6b9ffff640 1 --2- 172.21.15.83:0/4087154445 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f6ba012b490 0x7f6ba01c0f60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:20.285 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.281+0000 7f6b9ffff640 1 -- 172.21.15.83:0/4087154445 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f6ba006eb10 con 0x7f6ba00953a0 2024-09-14T03:40:20.286 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.281+0000 7f6ba4dd6640 1 --2- 172.21.15.83:0/4087154445 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f6ba012bdb0 0x7f6ba0130370 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:40:20.286 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.281+0000 7f6b9ffff640 1 --2- 172.21.15.83:0/4087154445 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f6ba00953a0 0x7f6ba01c0a20 secure :-1 s=READY pgs=25 cs=0 l=1 rev1=1 crypto rx=0x7f6b90005c80 tx=0x7f6b9000ef70 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:40:20.287 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.282+0000 7f6b9cff9640 1 -- 172.21.15.83:0/4087154445 <== mon.2 v2:172.21.15.83:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6b9000cd40 con 0x7f6ba00953a0 2024-09-14T03:40:20.287 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.282+0000 7f6b9cff9640 1 -- 172.21.15.83:0/4087154445 <== mon.2 v2:172.21.15.83:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f6b90004030 con 0x7f6ba00953a0 2024-09-14T03:40:20.287 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.282+0000 7f6ba6839640 1 -- 172.21.15.83:0/4087154445 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f6ba0132160 con 0x7f6ba00953a0 2024-09-14T03:40:20.287 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.282+0000 7f6b9cff9640 1 -- 172.21.15.83:0/4087154445 <== mon.2 v2:172.21.15.83:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6b90004480 con 0x7f6ba00953a0 2024-09-14T03:40:20.287 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.282+0000 7f6ba6839640 1 -- 172.21.15.83:0/4087154445 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7f6ba01308b0 con 0x7f6ba00953a0 2024-09-14T03:40:20.287 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.283+0000 7f6ba6839640 1 -- 172.21.15.83:0/4087154445 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f6ba00917d0 con 0x7f6ba00953a0 2024-09-14T03:40:20.287 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.283+0000 7f6b9cff9640 1 -- 172.21.15.83:0/4087154445 <== mon.2 v2:172.21.15.83:3302/0 4 ==== mgrmap(e 7) ==== 63244+0+0 (secure 0 0 0) 0x7f6b9001d020 con 0x7f6ba00953a0 2024-09-14T03:40:20.287 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.283+0000 7f6b9cff9640 1 -- 172.21.15.83:0/4087154445 <== mon.2 v2:172.21.15.83:3302/0 5 ==== osd_map(21..21 src has 1..21) ==== 3027+0+0 (secure 0 0 0) 0x7f6b90068cd0 con 0x7f6ba00953a0 2024-09-14T03:40:20.290 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.287+0000 7f6b9cff9640 1 -- 172.21.15.83:0/4087154445 <== mon.2 v2:172.21.15.83:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+152684 (secure 0 0 0) 0x7f6ba00917d0 con 0x7f6ba00953a0 2024-09-14T03:40:20.385 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.383+0000 7f061652b640 1 -- 172.21.15.83:0/3593692572 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f0610098d40 msgr2=0x7f0610099120 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:20.386 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.383+0000 7f061652b640 1 --2- 172.21.15.83:0/3593692572 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f0610098d40 0x7f0610099120 secure :-1 s=READY pgs=26 cs=0 l=1 rev1=1 crypto rx=0x7f0600009830 tx=0x7f060002a3f0 comp rx=0 tx=0).stop 2024-09-14T03:40:20.386 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.383+0000 7f061652b640 1 -- 172.21.15.83:0/3593692572 shutdown_connections 2024-09-14T03:40:20.386 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.383+0000 7f061652b640 1 --2- 172.21.15.83:0/3593692572 >> v2:172.21.15.83:3300/0 conn(0x7f0610099660 0x7f061012eab0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:20.386 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.383+0000 7f061652b640 1 --2- 172.21.15.83:0/3593692572 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f0610098d40 0x7f0610099120 unknown :-1 s=CLOSED pgs=26 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:20.386 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.383+0000 7f061652b640 1 -- 172.21.15.83:0/3593692572 >> 172.21.15.83:0/3593692572 conn(0x7f06100a1790 msgr2=0x7f06100a1ba0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:20.386 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.383+0000 7f061652b640 1 -- 172.21.15.83:0/3593692572 shutdown_connections 2024-09-14T03:40:20.386 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.383+0000 7f061652b640 1 -- 172.21.15.83:0/3593692572 wait complete. 2024-09-14T03:40:20.389 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.384+0000 7f061652b640 1 Processor -- start 2024-09-14T03:40:20.389 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.384+0000 7f061652b640 1 -- start start 2024-09-14T03:40:20.389 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.385+0000 7f061652b640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f0610098d40 0x7f06101c4f30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:20.389 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.385+0000 7f061652b640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f0610099660 0x7f06101c5470 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:20.389 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.385+0000 7f061652b640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f061012eff0 0x7f0610071e90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:20.389 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.385+0000 7f060f7fe640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f0610099660 0x7f06101c5470 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:20.389 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.385+0000 7f0614ac8640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f061012eff0 0x7f0610071e90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:20.389 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.385+0000 7f060f7fe640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f0610099660 0x7f06101c5470 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3301/0 says I am v2:172.21.15.83:39376/0 (socket says 172.21.15.83:39376) 2024-09-14T03:40:20.389 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.385+0000 7f060ffff640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f0610098d40 0x7f06101c4f30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:20.389 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.385+0000 7f0614ac8640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f061012eff0 0x7f0610071e90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3300/0 says I am v2:172.21.15.83:48552/0 (socket says 172.21.15.83:48552) 2024-09-14T03:40:20.389 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.385+0000 7f061652b640 1 -- 172.21.15.83:0/3583492613 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7f0610137870 con 0x7f061012eff0 2024-09-14T03:40:20.389 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.385+0000 7f060f7fe640 1 -- 172.21.15.83:0/3583492613 learned_addr learned my addr 172.21.15.83:0/3583492613 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:40:20.390 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.385+0000 7f061652b640 1 -- 172.21.15.83:0/3583492613 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7f06101376f0 con 0x7f0610099660 2024-09-14T03:40:20.390 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.385+0000 7f061652b640 1 -- 172.21.15.83:0/3583492613 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7f06101379f0 con 0x7f0610098d40 2024-09-14T03:40:20.390 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.385+0000 7f060f7fe640 1 -- 172.21.15.83:0/3583492613 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f0610098d40 msgr2=0x7f06101c4f30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:20.390 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.385+0000 7f060f7fe640 1 --2- 172.21.15.83:0/3583492613 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f0610098d40 0x7f06101c4f30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:20.390 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.385+0000 7f060f7fe640 1 -- 172.21.15.83:0/3583492613 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f061012eff0 msgr2=0x7f0610071e90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:20.390 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.385+0000 7f060f7fe640 1 --2- 172.21.15.83:0/3583492613 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f061012eff0 0x7f0610071e90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:20.390 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.385+0000 7f060f7fe640 1 -- 172.21.15.83:0/3583492613 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f0610129460 con 0x7f0610099660 2024-09-14T03:40:20.390 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.385+0000 7f0614ac8640 1 --2- 172.21.15.83:0/3583492613 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f061012eff0 0x7f0610071e90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:40:20.390 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.385+0000 7f060ffff640 1 --2- 172.21.15.83:0/3583492613 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f0610098d40 0x7f06101c4f30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:40:20.390 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.386+0000 7f060f7fe640 1 --2- 172.21.15.83:0/3583492613 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f0610099660 0x7f06101c5470 secure :-1 s=READY pgs=25 cs=0 l=1 rev1=1 crypto rx=0x7f060400ea90 tx=0x7f060400ef50 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:40:20.391 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.386+0000 7f565b4d4640 1 -- 172.21.15.83:0/2531258706 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f565412bd60 msgr2=0x7f565412c1d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:20.392 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.386+0000 7f565b4d4640 1 --2- 172.21.15.83:0/2531258706 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f565412bd60 0x7f565412c1d0 secure :-1 s=READY pgs=27 cs=0 l=1 rev1=1 crypto rx=0x7f564c009be0 tx=0x7f564c02a710 comp rx=0 tx=0).stop 2024-09-14T03:40:20.392 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.386+0000 7f565b4d4640 1 -- 172.21.15.83:0/2531258706 shutdown_connections 2024-09-14T03:40:20.392 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.386+0000 7f565b4d4640 1 --2- 172.21.15.83:0/2531258706 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f5654093140 0x7f565412b820 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:20.392 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.386+0000 7f565b4d4640 1 --2- 172.21.15.83:0/2531258706 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f565412bd60 0x7f565412c1d0 unknown :-1 s=CLOSED pgs=27 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:20.392 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.386+0000 7f565b4d4640 1 --2- 172.21.15.83:0/2531258706 >> v2:172.21.15.83:3300/0 conn(0x7f5654092820 0x7f5654092c00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:20.392 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.386+0000 7f565b4d4640 1 -- 172.21.15.83:0/2531258706 >> 172.21.15.83:0/2531258706 conn(0x7f565409e490 msgr2=0x7f565409e8a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:20.392 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.386+0000 7f060cff9640 1 -- 172.21.15.83:0/3583492613 <== mon.1 v2:172.21.15.83:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f060400ce40 con 0x7f0610099660 2024-09-14T03:40:20.393 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.386+0000 7f80a495f640 1 -- 172.21.15.83:0/3308769 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f80a0097740 msgr2=0x7f80a0133f80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:20.393 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.386+0000 7f80a495f640 1 --2- 172.21.15.83:0/3308769 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f80a0097740 0x7f80a0133f80 secure :-1 s=READY pgs=24 cs=0 l=1 rev1=1 crypto rx=0x7f8098009c10 tx=0x7f809802a780 comp rx=0 tx=0).stop 2024-09-14T03:40:20.393 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.386+0000 7f565b4d4640 1 -- 172.21.15.83:0/2531258706 shutdown_connections 2024-09-14T03:40:20.394 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.386+0000 7f060cff9640 1 -- 172.21.15.83:0/3583492613 <== mon.1 v2:172.21.15.83:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f0604004020 con 0x7f0610099660 2024-09-14T03:40:20.394 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.386+0000 7f061652b640 1 -- 172.21.15.83:0/3583492613 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f06101c59b0 con 0x7f0610099660 2024-09-14T03:40:20.394 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.386+0000 7f060cff9640 1 -- 172.21.15.83:0/3583492613 <== mon.1 v2:172.21.15.83:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0604004470 con 0x7f0610099660 2024-09-14T03:40:20.394 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.386+0000 7f061652b640 1 -- 172.21.15.83:0/3583492613 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7f061012f6b0 con 0x7f0610099660 2024-09-14T03:40:20.394 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.386+0000 7f80a495f640 1 -- 172.21.15.83:0/3308769 shutdown_connections 2024-09-14T03:40:20.395 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.386+0000 7f80a495f640 1 --2- 172.21.15.83:0/3308769 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f80a0097740 0x7f80a0133f80 unknown :-1 s=CLOSED pgs=24 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:20.395 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.386+0000 7f80a495f640 1 --2- 172.21.15.83:0/3308769 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f80a0070660 0x7f80a0070a40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:20.395 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.386+0000 7f80a495f640 1 -- 172.21.15.83:0/3308769 >> 172.21.15.83:0/3308769 conn(0x7f80a009dde0 msgr2=0x7f80a009e1f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:20.395 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.386+0000 7f565b4d4640 1 -- 172.21.15.83:0/2531258706 wait complete. 2024-09-14T03:40:20.395 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.386+0000 7f80a495f640 1 -- 172.21.15.83:0/3308769 shutdown_connections 2024-09-14T03:40:20.395 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.386+0000 7f80a495f640 1 -- 172.21.15.83:0/3308769 wait complete. 2024-09-14T03:40:20.396 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.387+0000 7f565b4d4640 1 Processor -- start 2024-09-14T03:40:20.396 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.387+0000 7f565b4d4640 1 -- start start 2024-09-14T03:40:20.396 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.387+0000 7f061652b640 1 -- 172.21.15.83:0/3583492613 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f061009d540 con 0x7f0610099660 2024-09-14T03:40:20.396 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.388+0000 7f060cff9640 1 -- 172.21.15.83:0/3583492613 <== mon.1 v2:172.21.15.83:3301/0 4 ==== mgrmap(e 7) ==== 63244+0+0 (secure 0 0 0) 0x7f06040035f0 con 0x7f0610099660 2024-09-14T03:40:20.397 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.388+0000 7f565b4d4640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f5654092820 0x7f565406f710 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:20.397 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.388+0000 7f80a495f640 1 Processor -- start 2024-09-14T03:40:20.397 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.388+0000 7f565b4d4640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f5654093140 0x7f565406fc50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:20.397 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.388+0000 7f80a495f640 1 -- start start 2024-09-14T03:40:20.398 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.388+0000 7f565b4d4640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f565412bd60 0x7f5654070190 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:20.398 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.388+0000 7f565b4d4640 1 -- --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7f565406eb90 con 0x7f565412bd60 2024-09-14T03:40:20.398 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.388+0000 7f565b4d4640 1 -- --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7f565406ea10 con 0x7f5654092820 2024-09-14T03:40:20.398 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.388+0000 7f565b4d4640 1 -- --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7f565406ed10 con 0x7f5654093140 2024-09-14T03:40:20.398 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.388+0000 7f060cff9640 1 -- 172.21.15.83:0/3583492613 <== mon.1 v2:172.21.15.83:3301/0 5 ==== osd_map(21..21 src has 1..21) ==== 3027+0+0 (secure 0 0 0) 0x7f06040695c0 con 0x7f0610099660 2024-09-14T03:40:20.399 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.388+0000 7f5659a71640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f565412bd60 0x7f5654070190 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:20.399 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.388+0000 7f5659a71640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f565412bd60 0x7f5654070190 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3300/0 says I am v2:172.21.15.83:48566/0 (socket says 172.21.15.83:48566) 2024-09-14T03:40:20.399 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.388+0000 7f5659a71640 1 -- 172.21.15.83:0/1754227062 learned_addr learned my addr 172.21.15.83:0/1754227062 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:40:20.399 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.388+0000 7f5659270640 1 --2- 172.21.15.83:0/1754227062 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f5654092820 0x7f565406f710 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:20.399 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.388+0000 7f5659a71640 1 -- 172.21.15.83:0/1754227062 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f5654093140 msgr2=0x7f565406fc50 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-09-14T03:40:20.399 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.388+0000 7f5659a71640 1 --2- 172.21.15.83:0/1754227062 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f5654093140 0x7f565406fc50 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:20.399 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.388+0000 7f5659a71640 1 -- 172.21.15.83:0/1754227062 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f5654092820 msgr2=0x7f565406f710 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:20.399 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.388+0000 7f5659a71640 1 --2- 172.21.15.83:0/1754227062 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f5654092820 0x7f565406f710 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:20.400 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.388+0000 7f5659a71640 1 -- 172.21.15.83:0/1754227062 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f56540713d0 con 0x7f565412bd60 2024-09-14T03:40:20.400 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.389+0000 7f5659a71640 1 --2- 172.21.15.83:0/1754227062 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f565412bd60 0x7f5654070190 secure :-1 s=READY pgs=36 cs=0 l=1 rev1=1 crypto rx=0x7f564c006010 tx=0x7f564c003e60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:40:20.400 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.389+0000 7f80a495f640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f80a0070660 0x7f80a01c2000 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:20.400 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.389+0000 7f5649ffb640 1 -- 172.21.15.83:0/1754227062 <== mon.0 v2:172.21.15.83:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f564c040020 con 0x7f565412bd60 2024-09-14T03:40:20.400 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.389+0000 7f80a495f640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f80a0070f80 0x7f80a01c2540 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:20.401 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.389+0000 7f565b4d4640 1 -- 172.21.15.83:0/1754227062 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f56540727c0 con 0x7f565412bd60 2024-09-14T03:40:20.401 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.389+0000 7f80a495f640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f80a0097740 0x7f80a0125ce0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:20.401 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.389+0000 7f5649ffb640 1 -- 172.21.15.83:0/1754227062 <== mon.0 v2:172.21.15.83:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f564c004040 con 0x7f565412bd60 2024-09-14T03:40:20.402 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.389+0000 7f80a495f640 1 -- --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7f80a0134940 con 0x7f80a0070f80 2024-09-14T03:40:20.402 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.389+0000 7f80a495f640 1 -- --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7f80a01347c0 con 0x7f80a0070660 2024-09-14T03:40:20.402 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.389+0000 7f80a495f640 1 -- --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7f80a0134ac0 con 0x7f80a0097740 2024-09-14T03:40:20.402 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.389+0000 7f809e59c640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f80a0070660 0x7f80a01c2000 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:20.402 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.389+0000 7f5649ffb640 1 -- 172.21.15.83:0/1754227062 <== mon.0 v2:172.21.15.83:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f564c036040 con 0x7f565412bd60 2024-09-14T03:40:20.402 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.389+0000 7f565b4d4640 1 -- 172.21.15.83:0/1754227062 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f5654072a80 con 0x7f565412bd60 2024-09-14T03:40:20.404 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.389+0000 7f809e59c640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f80a0070660 0x7f80a01c2000 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3301/0 says I am v2:172.21.15.83:39392/0 (socket says 172.21.15.83:39392) 2024-09-14T03:40:20.404 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.389+0000 7f809e59c640 1 -- 172.21.15.83:0/2002919364 learned_addr learned my addr 172.21.15.83:0/2002919364 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:40:20.404 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.389+0000 7f809dd9b640 1 --2- 172.21.15.83:0/2002919364 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f80a0070f80 0x7f80a01c2540 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:20.404 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.389+0000 7f809ed9d640 1 --2- 172.21.15.83:0/2002919364 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f80a0097740 0x7f80a0125ce0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:20.404 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.390+0000 7f809e59c640 1 -- 172.21.15.83:0/2002919364 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f80a0097740 msgr2=0x7f80a0125ce0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:20.404 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.390+0000 7f809e59c640 1 --2- 172.21.15.83:0/2002919364 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f80a0097740 0x7f80a0125ce0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:20.404 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.390+0000 7f809e59c640 1 -- 172.21.15.83:0/2002919364 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f80a0070f80 msgr2=0x7f80a01c2540 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:20.404 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.390+0000 7f809e59c640 1 --2- 172.21.15.83:0/2002919364 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f80a0070f80 0x7f80a01c2540 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:20.404 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.390+0000 7f809e59c640 1 -- 172.21.15.83:0/2002919364 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f80a00974f0 con 0x7f80a0070660 2024-09-14T03:40:20.404 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.390+0000 7f809dd9b640 1 --2- 172.21.15.83:0/2002919364 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f80a0070f80 0x7f80a01c2540 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-09-14T03:40:20.404 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.390+0000 7f809e59c640 1 --2- 172.21.15.83:0/2002919364 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f80a0070660 0x7f80a01c2000 secure :-1 s=READY pgs=26 cs=0 l=1 rev1=1 crypto rx=0x7f809000ead0 tx=0x7f809000ef90 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:40:20.405 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.390+0000 7f565b4d4640 1 -- 172.21.15.83:0/1754227062 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f565412b820 con 0x7f565412bd60 2024-09-14T03:40:20.405 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.390+0000 7f5649ffb640 1 -- 172.21.15.83:0/1754227062 <== mon.0 v2:172.21.15.83:3300/0 4 ==== mgrmap(e 7) ==== 63244+0+0 (secure 0 0 0) 0x7f564c043020 con 0x7f565412bd60 2024-09-14T03:40:20.405 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.391+0000 7f5649ffb640 1 -- 172.21.15.83:0/1754227062 <== mon.0 v2:172.21.15.83:3300/0 5 ==== osd_map(21..21 src has 1..21) ==== 3027+0+0 (secure 0 0 0) 0x7f564c089590 con 0x7f565412bd60 2024-09-14T03:40:20.405 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.394+0000 7f5649ffb640 1 -- 172.21.15.83:0/1754227062 <== mon.0 v2:172.21.15.83:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+152684 (secure 0 0 0) 0x7f564c04a030 con 0x7f565412bd60 2024-09-14T03:40:20.405 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.396+0000 7f060cff9640 1 -- 172.21.15.83:0/3583492613 <== mon.1 v2:172.21.15.83:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+152684 (secure 0 0 0) 0x7f061009d540 con 0x7f0610099660 2024-09-14T03:40:20.406 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.396+0000 7f808effd640 1 -- 172.21.15.83:0/2002919364 <== mon.1 v2:172.21.15.83:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f809000bc90 con 0x7f80a0070660 2024-09-14T03:40:20.406 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.396+0000 7f80a495f640 1 -- 172.21.15.83:0/2002919364 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f80a0128240 con 0x7f80a0070660 2024-09-14T03:40:20.406 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.396+0000 7f808effd640 1 -- 172.21.15.83:0/2002919364 <== mon.1 v2:172.21.15.83:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f809000be30 con 0x7f80a0070660 2024-09-14T03:40:20.407 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.396+0000 7f80a495f640 1 -- 172.21.15.83:0/2002919364 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7f80a01286b0 con 0x7f80a0070660 2024-09-14T03:40:20.407 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.396+0000 7f808effd640 1 -- 172.21.15.83:0/2002919364 <== mon.1 v2:172.21.15.83:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8090025060 con 0x7f80a0070660 2024-09-14T03:40:20.407 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.397+0000 7f80a495f640 1 -- 172.21.15.83:0/2002919364 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f80a0070a40 con 0x7f80a0070660 2024-09-14T03:40:20.407 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.397+0000 7f808effd640 1 -- 172.21.15.83:0/2002919364 <== mon.1 v2:172.21.15.83:3301/0 4 ==== mgrmap(e 7) ==== 63244+0+0 (secure 0 0 0) 0x7f8090004220 con 0x7f80a0070660 2024-09-14T03:40:20.407 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.398+0000 7f808effd640 1 -- 172.21.15.83:0/2002919364 <== mon.1 v2:172.21.15.83:3301/0 5 ==== osd_map(21..21 src has 1..21) ==== 3027+0+0 (secure 0 0 0) 0x7f809006aa80 con 0x7f80a0070660 2024-09-14T03:40:20.407 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.404+0000 7f808effd640 1 -- 172.21.15.83:0/2002919364 <== mon.1 v2:172.21.15.83:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+152684 (secure 0 0 0) 0x7f80900698d0 con 0x7f80a0070660 2024-09-14T03:40:20.494 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.492+0000 7f6ba6839640 1 -- 172.21.15.83:0/4087154445 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 0} v 0) -- 0x7f6ba0095810 con 0x7f6ba00953a0 2024-09-14T03:40:20.494 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.492+0000 7f6b9cff9640 1 -- 172.21.15.83:0/4087154445 <== mon.2 v2:172.21.15.83:3302/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 0}]=0 v0) ==== 74+0+12 (secure 0 0 0) 0x7f6b9000a620 con 0x7f6ba00953a0 2024-09-14T03:40:20.495 INFO:teuthology.orchestra.run.smithi083.stdout:38654705681 2024-09-14T03:40:20.496 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.494+0000 7f6ba6839640 1 -- 172.21.15.83:0/4087154445 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f6ba00953a0 msgr2=0x7f6ba01c0a20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:20.497 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.494+0000 7f6ba6839640 1 --2- 172.21.15.83:0/4087154445 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f6ba00953a0 0x7f6ba01c0a20 secure :-1 s=READY pgs=25 cs=0 l=1 rev1=1 crypto rx=0x7f6b90005c80 tx=0x7f6b9000ef70 comp rx=0 tx=0).stop 2024-09-14T03:40:20.497 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.494+0000 7f6ba6839640 1 -- 172.21.15.83:0/4087154445 shutdown_connections 2024-09-14T03:40:20.497 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.494+0000 7f6ba6839640 1 --2- 172.21.15.83:0/4087154445 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f6ba012bdb0 0x7f6ba0130370 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:20.497 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.494+0000 7f6ba6839640 1 --2- 172.21.15.83:0/4087154445 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f6ba012b490 0x7f6ba01c0f60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:20.497 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.494+0000 7f6ba6839640 1 --2- 172.21.15.83:0/4087154445 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f6ba00953a0 0x7f6ba01c0a20 unknown :-1 s=CLOSED pgs=25 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:20.497 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.494+0000 7f6ba6839640 1 -- 172.21.15.83:0/4087154445 >> 172.21.15.83:0/4087154445 conn(0x7f6ba009afd0 msgr2=0x7f6ba00982f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:20.497 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.494+0000 7f6ba6839640 1 -- 172.21.15.83:0/4087154445 shutdown_connections 2024-09-14T03:40:20.497 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.494+0000 7f6ba6839640 1 -- 172.21.15.83:0/4087154445 wait complete. 2024-09-14T03:40:20.507 INFO:tasks.ceph.ceph_manager.ceph:need seq 38654705682 got 38654705681 for osd.0 2024-09-14T03:40:20.547 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.544+0000 7f565b4d4640 1 -- 172.21.15.83:0/1754227062 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 3} v 0) -- 0x7f5654071b50 con 0x7f565412bd60 2024-09-14T03:40:20.548 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.545+0000 7f5649ffb640 1 -- 172.21.15.83:0/1754227062 <== mon.0 v2:172.21.15.83:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 3}]=0 v0) ==== 74+0+12 (secure 0 0 0) 0x7f564c034120 con 0x7f565412bd60 2024-09-14T03:40:20.548 INFO:teuthology.orchestra.run.smithi083.stdout:47244640273 2024-09-14T03:40:20.548 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.545+0000 7f061652b640 1 -- 172.21.15.83:0/3583492613 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 1} v 0) -- 0x7f0610099120 con 0x7f0610099660 2024-09-14T03:40:20.548 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.545+0000 7f060cff9640 1 -- 172.21.15.83:0/3583492613 <== mon.1 v2:172.21.15.83:3301/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 1}]=0 v0) ==== 74+0+12 (secure 0 0 0) 0x7f0604068410 con 0x7f0610099660 2024-09-14T03:40:20.549 INFO:teuthology.orchestra.run.smithi083.stdout:47244640273 2024-09-14T03:40:20.550 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.547+0000 7f061652b640 1 -- 172.21.15.83:0/3583492613 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f0610099660 msgr2=0x7f06101c5470 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:20.550 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.547+0000 7f061652b640 1 --2- 172.21.15.83:0/3583492613 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f0610099660 0x7f06101c5470 secure :-1 s=READY pgs=25 cs=0 l=1 rev1=1 crypto rx=0x7f060400ea90 tx=0x7f060400ef50 comp rx=0 tx=0).stop 2024-09-14T03:40:20.550 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.547+0000 7f061652b640 1 -- 172.21.15.83:0/3583492613 shutdown_connections 2024-09-14T03:40:20.550 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.547+0000 7f061652b640 1 --2- 172.21.15.83:0/3583492613 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f061012eff0 0x7f0610071e90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:20.550 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.547+0000 7f061652b640 1 --2- 172.21.15.83:0/3583492613 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f0610099660 0x7f06101c5470 unknown :-1 s=CLOSED pgs=25 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:20.551 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.547+0000 7f061652b640 1 --2- 172.21.15.83:0/3583492613 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f0610098d40 0x7f06101c4f30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:20.551 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.547+0000 7f061652b640 1 -- 172.21.15.83:0/3583492613 >> 172.21.15.83:0/3583492613 conn(0x7f06100a1790 msgr2=0x7f0610128440 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:20.551 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.548+0000 7f565b4d4640 1 -- 172.21.15.83:0/1754227062 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f565412bd60 msgr2=0x7f5654070190 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:20.551 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.548+0000 7f565b4d4640 1 --2- 172.21.15.83:0/1754227062 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f565412bd60 0x7f5654070190 secure :-1 s=READY pgs=36 cs=0 l=1 rev1=1 crypto rx=0x7f564c006010 tx=0x7f564c003e60 comp rx=0 tx=0).stop 2024-09-14T03:40:20.551 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.548+0000 7f061652b640 1 -- 172.21.15.83:0/3583492613 shutdown_connections 2024-09-14T03:40:20.552 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.548+0000 7f565b4d4640 1 -- 172.21.15.83:0/1754227062 shutdown_connections 2024-09-14T03:40:20.552 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.548+0000 7f565b4d4640 1 --2- 172.21.15.83:0/1754227062 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f565412bd60 0x7f5654070190 unknown :-1 s=CLOSED pgs=36 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:20.552 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.548+0000 7f565b4d4640 1 --2- 172.21.15.83:0/1754227062 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f5654093140 0x7f565406fc50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:20.552 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.548+0000 7f565b4d4640 1 --2- 172.21.15.83:0/1754227062 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f5654092820 0x7f565406f710 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:20.553 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.548+0000 7f565b4d4640 1 -- 172.21.15.83:0/1754227062 >> 172.21.15.83:0/1754227062 conn(0x7f565409e490 msgr2=0x7f5654125920 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:20.553 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.548+0000 7f061652b640 1 -- 172.21.15.83:0/3583492613 wait complete. 2024-09-14T03:40:20.553 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.548+0000 7f565b4d4640 1 -- 172.21.15.83:0/1754227062 shutdown_connections 2024-09-14T03:40:20.554 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.548+0000 7f565b4d4640 1 -- 172.21.15.83:0/1754227062 wait complete. 2024-09-14T03:40:20.557 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.555+0000 7f80a495f640 1 -- 172.21.15.83:0/2002919364 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 2} v 0) -- 0x7f80a00697f0 con 0x7f80a0070660 2024-09-14T03:40:20.557 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.555+0000 7f808effd640 1 -- 172.21.15.83:0/2002919364 <== mon.1 v2:172.21.15.83:3301/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 2}]=0 v0) ==== 74+0+12 (secure 0 0 0) 0x7f8090043820 con 0x7f80a0070660 2024-09-14T03:40:20.558 INFO:teuthology.orchestra.run.smithi083.stdout:47244640273 2024-09-14T03:40:20.559 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.557+0000 7f80a495f640 1 -- 172.21.15.83:0/2002919364 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f80a0070660 msgr2=0x7f80a01c2000 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:20.560 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.557+0000 7f80a495f640 1 --2- 172.21.15.83:0/2002919364 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f80a0070660 0x7f80a01c2000 secure :-1 s=READY pgs=26 cs=0 l=1 rev1=1 crypto rx=0x7f809000ead0 tx=0x7f809000ef90 comp rx=0 tx=0).stop 2024-09-14T03:40:20.560 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.557+0000 7f80a495f640 1 -- 172.21.15.83:0/2002919364 shutdown_connections 2024-09-14T03:40:20.560 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.557+0000 7f80a495f640 1 --2- 172.21.15.83:0/2002919364 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f80a0097740 0x7f80a0125ce0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:20.560 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.557+0000 7f80a495f640 1 --2- 172.21.15.83:0/2002919364 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f80a0070f80 0x7f80a01c2540 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:20.560 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.557+0000 7f80a495f640 1 --2- 172.21.15.83:0/2002919364 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f80a0070660 0x7f80a01c2000 unknown :-1 s=CLOSED pgs=26 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:20.560 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.557+0000 7f80a495f640 1 -- 172.21.15.83:0/2002919364 >> 172.21.15.83:0/2002919364 conn(0x7f80a009dde0 msgr2=0x7f80a012fad0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:20.560 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.557+0000 7f80a495f640 1 -- 172.21.15.83:0/2002919364 shutdown_connections 2024-09-14T03:40:20.560 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:20.557+0000 7f80a495f640 1 -- 172.21.15.83:0/2002919364 wait complete. 2024-09-14T03:40:20.565 INFO:tasks.ceph.ceph_manager.ceph:need seq 47244640274 got 47244640273 for osd.3 2024-09-14T03:40:20.566 INFO:tasks.ceph.ceph_manager.ceph:need seq 47244640274 got 47244640273 for osd.1 2024-09-14T03:40:20.570 INFO:tasks.ceph.ceph_manager.ceph:need seq 47244640274 got 47244640273 for osd.2 2024-09-14T03:40:21.038 INFO:tasks.ceph.mgr.y.smithi083.stderr:2024-09-14T03:40:21.035+0000 7ff1f31fa640 -1 mgr.server handle_report got status from non-daemon mon.a 2024-09-14T03:40:21.041 INFO:tasks.ceph.mgr.y.smithi083.stderr:2024-09-14T03:40:21.039+0000 7ff1f31fa640 -1 mgr.server handle_report got status from non-daemon mon.b 2024-09-14T03:40:21.042 INFO:tasks.ceph.mgr.y.smithi083.stderr:2024-09-14T03:40:21.040+0000 7ff1f31fa640 -1 mgr.server handle_report got status from non-daemon mon.c 2024-09-14T03:40:21.508 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.0 2024-09-14T03:40:21.566 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.3 2024-09-14T03:40:21.567 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.1 2024-09-14T03:40:21.571 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.2 2024-09-14T03:40:21.624 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.622+0000 7f8077273640 1 -- 172.21.15.83:0/2707690699 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f807006edc0 msgr2=0x7f8070131da0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:21.624 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.622+0000 7f8077273640 1 --2- 172.21.15.83:0/2707690699 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f807006edc0 0x7f8070131da0 secure :-1 s=READY pgs=27 cs=0 l=1 rev1=1 crypto rx=0x7f8060009900 tx=0x7f806002a4a0 comp rx=0 tx=0).stop 2024-09-14T03:40:21.624 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.622+0000 7f8077273640 1 -- 172.21.15.83:0/2707690699 shutdown_connections 2024-09-14T03:40:21.624 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.622+0000 7f8077273640 1 --2- 172.21.15.83:0/2707690699 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f80701322e0 0x7f80701346d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:21.624 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.622+0000 7f8077273640 1 --2- 172.21.15.83:0/2707690699 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f807006edc0 0x7f8070131da0 unknown :-1 s=CLOSED pgs=27 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:21.625 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.622+0000 7f8077273640 1 -- 172.21.15.83:0/2707690699 >> 172.21.15.83:0/2707690699 conn(0x7f807009e490 msgr2=0x7f807009e8a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:21.625 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.622+0000 7f8077273640 1 -- 172.21.15.83:0/2707690699 shutdown_connections 2024-09-14T03:40:21.625 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.622+0000 7f8077273640 1 -- 172.21.15.83:0/2707690699 wait complete. 2024-09-14T03:40:21.627 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.623+0000 7f8077273640 1 Processor -- start 2024-09-14T03:40:21.627 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.623+0000 7f8077273640 1 -- start start 2024-09-14T03:40:21.627 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.624+0000 7f8077273640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f807006e2d0 0x7f8070124090 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:21.627 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.624+0000 7f8077273640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f807006edc0 0x7f80701245d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:21.627 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.624+0000 7f8077273640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f80701322e0 0x7f8070124b10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:21.627 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.624+0000 7f8077273640 1 -- --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7f8070135780 con 0x7f80701322e0 2024-09-14T03:40:21.627 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.624+0000 7f8077273640 1 -- --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7f8070135600 con 0x7f807006e2d0 2024-09-14T03:40:21.627 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.624+0000 7f8077273640 1 -- --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7f8070135900 con 0x7f807006edc0 2024-09-14T03:40:21.627 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.624+0000 7f807500f640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f807006e2d0 0x7f8070124090 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:21.627 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.624+0000 7f807500f640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f807006e2d0 0x7f8070124090 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3301/0 says I am v2:172.21.15.83:39410/0 (socket says 172.21.15.83:39410) 2024-09-14T03:40:21.628 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.624+0000 7f807500f640 1 -- 172.21.15.83:0/3100784342 learned_addr learned my addr 172.21.15.83:0/3100784342 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:40:21.628 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.624+0000 7f8075810640 1 --2- 172.21.15.83:0/3100784342 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f80701322e0 0x7f8070124b10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:21.628 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.624+0000 7f807480e640 1 --2- 172.21.15.83:0/3100784342 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f807006edc0 0x7f80701245d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:21.628 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.625+0000 7f8075810640 1 -- 172.21.15.83:0/3100784342 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f807006edc0 msgr2=0x7f80701245d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:21.628 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.625+0000 7f8075810640 1 --2- 172.21.15.83:0/3100784342 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f807006edc0 0x7f80701245d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:21.628 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.625+0000 7f8075810640 1 -- 172.21.15.83:0/3100784342 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f807006e2d0 msgr2=0x7f8070124090 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:21.628 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.625+0000 7f8075810640 1 --2- 172.21.15.83:0/3100784342 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f807006e2d0 0x7f8070124090 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:21.628 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.625+0000 7f8075810640 1 -- 172.21.15.83:0/3100784342 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f8070125680 con 0x7f80701322e0 2024-09-14T03:40:21.628 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.625+0000 7f807480e640 1 --2- 172.21.15.83:0/3100784342 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f807006edc0 0x7f80701245d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:40:21.628 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.625+0000 7f8075810640 1 --2- 172.21.15.83:0/3100784342 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f80701322e0 0x7f8070124b10 secure :-1 s=READY pgs=38 cs=0 l=1 rev1=1 crypto rx=0x7f806800ec50 tx=0x7f806800c6a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:40:21.628 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.625+0000 7f8065ffb640 1 -- 172.21.15.83:0/3100784342 <== mon.0 v2:172.21.15.83:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8068003b70 con 0x7f80701322e0 2024-09-14T03:40:21.628 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.625+0000 7f8065ffb640 1 -- 172.21.15.83:0/3100784342 <== mon.0 v2:172.21.15.83:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f8068003d10 con 0x7f80701322e0 2024-09-14T03:40:21.629 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.625+0000 7f8077273640 1 -- 172.21.15.83:0/3100784342 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f8070125050 con 0x7f80701322e0 2024-09-14T03:40:21.629 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.625+0000 7f8065ffb640 1 -- 172.21.15.83:0/3100784342 <== mon.0 v2:172.21.15.83:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8068004280 con 0x7f80701322e0 2024-09-14T03:40:21.629 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.625+0000 7f8077273640 1 -- 172.21.15.83:0/3100784342 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f80701269a0 con 0x7f80701322e0 2024-09-14T03:40:21.629 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.627+0000 7f8077273640 1 -- 172.21.15.83:0/3100784342 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f80701266c0 con 0x7f80701322e0 2024-09-14T03:40:21.629 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.627+0000 7f8065ffb640 1 -- 172.21.15.83:0/3100784342 <== mon.0 v2:172.21.15.83:3300/0 4 ==== mgrmap(e 8) ==== 63336+0+0 (secure 0 0 0) 0x7f8068004420 con 0x7f80701322e0 2024-09-14T03:40:21.629 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.627+0000 7f8065ffb640 1 -- 172.21.15.83:0/3100784342 <== mon.0 v2:172.21.15.83:3300/0 5 ==== osd_map(21..21 src has 1..21) ==== 3027+0+0 (secure 0 0 0) 0x7f80680698e0 con 0x7f80701322e0 2024-09-14T03:40:21.636 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.635+0000 7f8065ffb640 1 -- 172.21.15.83:0/3100784342 <== mon.0 v2:172.21.15.83:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+157506 (secure 0 0 0) 0x7f8068004710 con 0x7f80701322e0 2024-09-14T03:40:21.679 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.677+0000 7fd77427a640 1 -- 172.21.15.83:0/132413546 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fd76c129510 msgr2=0x7fd76c1298f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:21.679 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.677+0000 7fd77427a640 1 --2- 172.21.15.83:0/132413546 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fd76c129510 0x7fd76c1298f0 secure :-1 s=READY pgs=28 cs=0 l=1 rev1=1 crypto rx=0x7fd768005fd0 tx=0x7fd76802a4b0 comp rx=0 tx=0).stop 2024-09-14T03:40:21.679 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.677+0000 7fd77427a640 1 -- 172.21.15.83:0/132413546 shutdown_connections 2024-09-14T03:40:21.679 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.677+0000 7fd77427a640 1 --2- 172.21.15.83:0/132413546 >> v2:172.21.15.83:3300/0 conn(0x7fd76c129e30 0x7fd76c12c820 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:21.679 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.677+0000 7fd77427a640 1 --2- 172.21.15.83:0/132413546 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fd76c129510 0x7fd76c1298f0 unknown :-1 s=CLOSED pgs=28 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:21.679 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.677+0000 7fd77427a640 1 -- 172.21.15.83:0/132413546 >> 172.21.15.83:0/132413546 conn(0x7fd76c09d370 msgr2=0x7fd76c09d780 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:21.679 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.677+0000 7fd77427a640 1 -- 172.21.15.83:0/132413546 shutdown_connections 2024-09-14T03:40:21.679 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.677+0000 7fd77427a640 1 -- 172.21.15.83:0/132413546 wait complete. 2024-09-14T03:40:21.680 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.678+0000 7fd77427a640 1 Processor -- start 2024-09-14T03:40:21.680 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.679+0000 7fd77427a640 1 -- start start 2024-09-14T03:40:21.682 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.679+0000 7fd77427a640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fd76c129510 0x7fd76c1c2e10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:21.682 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.679+0000 7fd77427a640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fd76c129e30 0x7fd76c1c3350 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:21.682 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.679+0000 7fd77427a640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fd76c12cd60 0x7fd76c12fe30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:21.682 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.679+0000 7fd77427a640 1 -- --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7fd76c135750 con 0x7fd76c129e30 2024-09-14T03:40:21.683 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.679+0000 7fd77427a640 1 -- --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7fd76c1355d0 con 0x7fd76c129510 2024-09-14T03:40:21.683 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.679+0000 7fd77427a640 1 -- --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7fd76c1358d0 con 0x7fd76c12cd60 2024-09-14T03:40:21.683 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.679+0000 7fd771815640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fd76c129e30 0x7fd76c1c3350 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:21.683 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.679+0000 7fd771815640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fd76c129e30 0x7fd76c1c3350 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3300/0 says I am v2:172.21.15.83:48598/0 (socket says 172.21.15.83:48598) 2024-09-14T03:40:21.683 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.679+0000 7fd771815640 1 -- 172.21.15.83:0/618193875 learned_addr learned my addr 172.21.15.83:0/618193875 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:40:21.683 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.679+0000 7fd772016640 1 --2- 172.21.15.83:0/618193875 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fd76c129510 0x7fd76c1c2e10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:21.683 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.679+0000 7fd772817640 1 --2- 172.21.15.83:0/618193875 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fd76c12cd60 0x7fd76c12fe30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:21.683 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.680+0000 7fd772817640 1 -- 172.21.15.83:0/618193875 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fd76c129510 msgr2=0x7fd76c1c2e10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:21.683 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.680+0000 7fd772817640 1 --2- 172.21.15.83:0/618193875 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fd76c129510 0x7fd76c1c2e10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:21.684 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.680+0000 7fd772817640 1 -- 172.21.15.83:0/618193875 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fd76c129e30 msgr2=0x7fd76c1c3350 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:21.684 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.680+0000 7fd772817640 1 --2- 172.21.15.83:0/618193875 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fd76c129e30 0x7fd76c1c3350 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:21.684 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.680+0000 7fd772817640 1 -- 172.21.15.83:0/618193875 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fd76c1317c0 con 0x7fd76c12cd60 2024-09-14T03:40:21.684 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.680+0000 7fd771815640 1 --2- 172.21.15.83:0/618193875 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fd76c129e30 0x7fd76c1c3350 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:40:21.684 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.680+0000 7fd772016640 1 --2- 172.21.15.83:0/618193875 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fd76c129510 0x7fd76c1c2e10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:40:21.684 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.680+0000 7fd772817640 1 --2- 172.21.15.83:0/618193875 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fd76c12cd60 0x7fd76c12fe30 secure :-1 s=READY pgs=29 cs=0 l=1 rev1=1 crypto rx=0x7fd76400c790 tx=0x7fd76400cc50 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:40:21.684 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.680+0000 7fd762ffd640 1 -- 172.21.15.83:0/618193875 <== mon.2 v2:172.21.15.83:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd764003b70 con 0x7fd76c12cd60 2024-09-14T03:40:21.684 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.681+0000 7fd762ffd640 1 -- 172.21.15.83:0/618193875 <== mon.2 v2:172.21.15.83:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fd764003d10 con 0x7fd76c12cd60 2024-09-14T03:40:21.684 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.681+0000 7fd77427a640 1 -- 172.21.15.83:0/618193875 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fd76c132980 con 0x7fd76c12cd60 2024-09-14T03:40:21.684 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.681+0000 7fd762ffd640 1 -- 172.21.15.83:0/618193875 <== mon.2 v2:172.21.15.83:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd76400aa30 con 0x7fd76c12cd60 2024-09-14T03:40:21.684 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.681+0000 7fd77427a640 1 -- 172.21.15.83:0/618193875 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7fd76c1c3890 con 0x7fd76c12cd60 2024-09-14T03:40:21.685 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.682+0000 7fd77427a640 1 -- 172.21.15.83:0/618193875 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fd76c127ed0 con 0x7fd76c12cd60 2024-09-14T03:40:21.685 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.683+0000 7fd762ffd640 1 -- 172.21.15.83:0/618193875 <== mon.2 v2:172.21.15.83:3302/0 4 ==== mgrmap(e 8) ==== 63336+0+0 (secure 0 0 0) 0x7fd76400abd0 con 0x7fd76c12cd60 2024-09-14T03:40:21.685 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.683+0000 7fd762ffd640 1 -- 172.21.15.83:0/618193875 <== mon.2 v2:172.21.15.83:3302/0 5 ==== osd_map(21..21 src has 1..21) ==== 3027+0+0 (secure 0 0 0) 0x7fd764068990 con 0x7fd76c12cd60 2024-09-14T03:40:21.688 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.687+0000 7fd762ffd640 1 -- 172.21.15.83:0/618193875 <== mon.2 v2:172.21.15.83:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+157506 (secure 0 0 0) 0x7fd76c127ed0 con 0x7fd76c12cd60 2024-09-14T03:40:21.694 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.693+0000 7f2a567a1640 1 -- 172.21.15.83:0/420245133 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f2a50129e30 msgr2=0x7f2a5012c820 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:21.695 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.693+0000 7f2a567a1640 1 --2- 172.21.15.83:0/420245133 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f2a50129e30 0x7f2a5012c820 secure :-1 s=READY pgs=28 cs=0 l=1 rev1=1 crypto rx=0x7f2a44009860 tx=0x7f2a4402a3d0 comp rx=0 tx=0).stop 2024-09-14T03:40:21.695 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.693+0000 7f2a567a1640 1 -- 172.21.15.83:0/420245133 shutdown_connections 2024-09-14T03:40:21.695 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.693+0000 7f2a567a1640 1 --2- 172.21.15.83:0/420245133 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f2a50129e30 0x7f2a5012c820 unknown :-1 s=CLOSED pgs=28 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:21.695 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.693+0000 7f2a567a1640 1 --2- 172.21.15.83:0/420245133 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f2a50129510 0x7f2a501298f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:21.695 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.693+0000 7f2a567a1640 1 -- 172.21.15.83:0/420245133 >> 172.21.15.83:0/420245133 conn(0x7f2a5009d370 msgr2=0x7f2a5009d780 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:21.695 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.693+0000 7f2a567a1640 1 -- 172.21.15.83:0/420245133 shutdown_connections 2024-09-14T03:40:21.695 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.693+0000 7f2a567a1640 1 -- 172.21.15.83:0/420245133 wait complete. 2024-09-14T03:40:21.696 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.694+0000 7f2a567a1640 1 Processor -- start 2024-09-14T03:40:21.696 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.694+0000 7f2a567a1640 1 -- start start 2024-09-14T03:40:21.696 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.694+0000 7f2a567a1640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f2a50129510 0x7f2a50132bd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:21.696 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.695+0000 7f2a567a1640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f2a50129e30 0x7f2a50133110 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:21.696 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.695+0000 7f2a567a1640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f2a5012cd60 0x7f2a50133650 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:21.696 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.695+0000 7f2a567a1640 1 -- --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7f2a501357e0 con 0x7f2a5012cd60 2024-09-14T03:40:21.697 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.695+0000 7f2a567a1640 1 -- --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7f2a50135660 con 0x7f2a50129e30 2024-09-14T03:40:21.697 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.695+0000 7f2a567a1640 1 -- --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7f2a50135960 con 0x7f2a50129510 2024-09-14T03:40:21.697 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.695+0000 7f2a4f7fe640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f2a50129e30 0x7f2a50133110 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:21.697 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.695+0000 7f2a4f7fe640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f2a50129e30 0x7f2a50133110 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3301/0 says I am v2:172.21.15.83:39426/0 (socket says 172.21.15.83:39426) 2024-09-14T03:40:21.697 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.695+0000 7f2a4f7fe640 1 -- 172.21.15.83:0/1165836833 learned_addr learned my addr 172.21.15.83:0/1165836833 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:40:21.697 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.695+0000 7f2a4ffff640 1 --2- 172.21.15.83:0/1165836833 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f2a50129510 0x7f2a50132bd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:21.697 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.695+0000 7f2a54d3e640 1 --2- 172.21.15.83:0/1165836833 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f2a5012cd60 0x7f2a50133650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:21.698 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.695+0000 7f2a4f7fe640 1 -- 172.21.15.83:0/1165836833 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f2a50129510 msgr2=0x7f2a50132bd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:21.698 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.695+0000 7f2a4f7fe640 1 --2- 172.21.15.83:0/1165836833 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f2a50129510 0x7f2a50132bd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:21.698 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.695+0000 7f2a4f7fe640 1 -- 172.21.15.83:0/1165836833 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f2a5012cd60 msgr2=0x7f2a50133650 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:21.698 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.695+0000 7f2a4f7fe640 1 --2- 172.21.15.83:0/1165836833 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f2a5012cd60 0x7f2a50133650 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:21.698 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.695+0000 7f2a4f7fe640 1 -- 172.21.15.83:0/1165836833 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f2a501348d0 con 0x7f2a50129e30 2024-09-14T03:40:21.698 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.695+0000 7f2a4f7fe640 1 --2- 172.21.15.83:0/1165836833 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f2a50129e30 0x7f2a50133110 secure :-1 s=READY pgs=29 cs=0 l=1 rev1=1 crypto rx=0x7f2a44000c00 tx=0x7f2a44032f80 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:40:21.698 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.696+0000 7f2a4cff9640 1 -- 172.21.15.83:0/1165836833 <== mon.1 v2:172.21.15.83:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2a44040070 con 0x7f2a50129e30 2024-09-14T03:40:21.698 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.696+0000 7f2a4cff9640 1 -- 172.21.15.83:0/1165836833 <== mon.1 v2:172.21.15.83:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f2a44003b10 con 0x7f2a50129e30 2024-09-14T03:40:21.698 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.696+0000 7f2a4cff9640 1 -- 172.21.15.83:0/1165836833 <== mon.1 v2:172.21.15.83:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2a44045460 con 0x7f2a50129e30 2024-09-14T03:40:21.698 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.696+0000 7f2a567a1640 1 -- 172.21.15.83:0/1165836833 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f2a501168f0 con 0x7f2a50129e30 2024-09-14T03:40:21.699 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.696+0000 7f2a567a1640 1 -- 172.21.15.83:0/1165836833 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7f2a50115f50 con 0x7f2a50129e30 2024-09-14T03:40:21.699 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.697+0000 7f2a4cff9640 1 -- 172.21.15.83:0/1165836833 <== mon.1 v2:172.21.15.83:3301/0 4 ==== mgrmap(e 8) ==== 63336+0+0 (secure 0 0 0) 0x7f2a44039040 con 0x7f2a50129e30 2024-09-14T03:40:21.699 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.697+0000 7f2a567a1640 1 -- 172.21.15.83:0/1165836833 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f2a50071ec0 con 0x7f2a50129e30 2024-09-14T03:40:21.699 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.698+0000 7f2a4cff9640 1 -- 172.21.15.83:0/1165836833 <== mon.1 v2:172.21.15.83:3301/0 5 ==== osd_map(21..21 src has 1..21) ==== 3027+0+0 (secure 0 0 0) 0x7f2a4408a0b0 con 0x7f2a50129e30 2024-09-14T03:40:21.706 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.705+0000 7f2a4cff9640 1 -- 172.21.15.83:0/1165836833 <== mon.1 v2:172.21.15.83:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+157506 (secure 0 0 0) 0x7f2a50071ec0 con 0x7f2a50129e30 2024-09-14T03:40:21.721 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.719+0000 7f0d09366640 1 -- 172.21.15.83:0/1443817574 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f0d04098d40 msgr2=0x7f0d04099120 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:21.721 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.719+0000 7f0d09366640 1 --2- 172.21.15.83:0/1443817574 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f0d04098d40 0x7f0d04099120 secure :-1 s=READY pgs=42 cs=0 l=1 rev1=1 crypto rx=0x7f0cf4005fd0 tx=0x7f0cf402a4b0 comp rx=0 tx=0).stop 2024-09-14T03:40:21.721 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.719+0000 7f0d09366640 1 -- 172.21.15.83:0/1443817574 shutdown_connections 2024-09-14T03:40:21.721 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.719+0000 7f0d09366640 1 --2- 172.21.15.83:0/1443817574 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f0d0412f1c0 0x7f0d0412f600 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:21.721 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.719+0000 7f0d09366640 1 --2- 172.21.15.83:0/1443817574 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f0d04098d40 0x7f0d04099120 unknown :-1 s=CLOSED pgs=42 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:21.721 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.719+0000 7f0d09366640 1 -- 172.21.15.83:0/1443817574 >> 172.21.15.83:0/1443817574 conn(0x7f0d040a1790 msgr2=0x7f0d040a1ba0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:21.722 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.719+0000 7f0d09366640 1 -- 172.21.15.83:0/1443817574 shutdown_connections 2024-09-14T03:40:21.722 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.719+0000 7f0d09366640 1 -- 172.21.15.83:0/1443817574 wait complete. 2024-09-14T03:40:21.722 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.721+0000 7f0d09366640 1 Processor -- start 2024-09-14T03:40:21.722 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.721+0000 7f0d09366640 1 -- start start 2024-09-14T03:40:21.724 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.722+0000 7f0d09366640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f0d04098d40 0x7f0d041c4d50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:21.725 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.722+0000 7f0d09366640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f0d04099660 0x7f0d041c5290 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:21.725 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.722+0000 7f0d09366640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f0d0412f1c0 0x7f0d04071320 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:21.725 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.722+0000 7f0d027fc640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f0d04099660 0x7f0d041c5290 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:21.725 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.722+0000 7f0d027fc640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f0d04099660 0x7f0d041c5290 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3302/0 says I am v2:172.21.15.83:58626/0 (socket says 172.21.15.83:58626) 2024-09-14T03:40:21.725 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.722+0000 7f0d02ffd640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f0d04098d40 0x7f0d041c4d50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:21.725 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.722+0000 7f0d02ffd640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f0d04098d40 0x7f0d041c4d50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3301/0 says I am v2:172.21.15.83:39454/0 (socket says 172.21.15.83:39454) 2024-09-14T03:40:21.725 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.722+0000 7f0d037fe640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f0d0412f1c0 0x7f0d04071320 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:21.725 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.722+0000 7f0d037fe640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f0d0412f1c0 0x7f0d04071320 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3300/0 says I am v2:172.21.15.83:48632/0 (socket says 172.21.15.83:48632) 2024-09-14T03:40:21.725 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.722+0000 7f0d027fc640 1 -- 172.21.15.83:0/3515576781 learned_addr learned my addr 172.21.15.83:0/3515576781 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:40:21.725 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.722+0000 7f0d09366640 1 -- 172.21.15.83:0/3515576781 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7f0d04137870 con 0x7f0d0412f1c0 2024-09-14T03:40:21.725 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.722+0000 7f0d09366640 1 -- 172.21.15.83:0/3515576781 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7f0d041376f0 con 0x7f0d04098d40 2024-09-14T03:40:21.725 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.722+0000 7f0d09366640 1 -- 172.21.15.83:0/3515576781 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7f0d041379f0 con 0x7f0d04099660 2024-09-14T03:40:21.726 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.723+0000 7f0d027fc640 1 -- 172.21.15.83:0/3515576781 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f0d04098d40 msgr2=0x7f0d041c4d50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:21.726 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.723+0000 7f0d027fc640 1 --2- 172.21.15.83:0/3515576781 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f0d04098d40 0x7f0d041c4d50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:21.726 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.723+0000 7f0d027fc640 1 -- 172.21.15.83:0/3515576781 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f0d0412f1c0 msgr2=0x7f0d04071320 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:21.726 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.723+0000 7f0d027fc640 1 --2- 172.21.15.83:0/3515576781 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f0d0412f1c0 0x7f0d04071320 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:21.726 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.723+0000 7f0d027fc640 1 -- 172.21.15.83:0/3515576781 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f0d04129460 con 0x7f0d04099660 2024-09-14T03:40:21.726 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.723+0000 7f0d037fe640 1 --2- 172.21.15.83:0/3515576781 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f0d0412f1c0 0x7f0d04071320 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-09-14T03:40:21.726 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.723+0000 7f0d02ffd640 1 --2- 172.21.15.83:0/3515576781 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f0d04098d40 0x7f0d041c4d50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-09-14T03:40:21.726 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.723+0000 7f0d027fc640 1 --2- 172.21.15.83:0/3515576781 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f0d04099660 0x7f0d041c5290 secure :-1 s=READY pgs=30 cs=0 l=1 rev1=1 crypto rx=0x7f0cf800c500 tx=0x7f0cf800c9c0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:40:21.726 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.723+0000 7f0ceb7fe640 1 -- 172.21.15.83:0/3515576781 <== mon.2 v2:172.21.15.83:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0cf80052d0 con 0x7f0d04099660 2024-09-14T03:40:21.726 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.723+0000 7f0ceb7fe640 1 -- 172.21.15.83:0/3515576781 <== mon.2 v2:172.21.15.83:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f0cf8005470 con 0x7f0d04099660 2024-09-14T03:40:21.726 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.723+0000 7f0d09366640 1 -- 172.21.15.83:0/3515576781 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f0d04074070 con 0x7f0d04099660 2024-09-14T03:40:21.726 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.723+0000 7f0ceb7fe640 1 -- 172.21.15.83:0/3515576781 <== mon.2 v2:172.21.15.83:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0cf8012870 con 0x7f0d04099660 2024-09-14T03:40:21.727 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.723+0000 7f0d09366640 1 -- 172.21.15.83:0/3515576781 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7f0d041c59b0 con 0x7f0d04099660 2024-09-14T03:40:21.727 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.724+0000 7f0d09366640 1 -- 172.21.15.83:0/3515576781 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f0d04099120 con 0x7f0d04099660 2024-09-14T03:40:21.727 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.725+0000 7f0ceb7fe640 1 -- 172.21.15.83:0/3515576781 <== mon.2 v2:172.21.15.83:3302/0 4 ==== mgrmap(e 8) ==== 63336+0+0 (secure 0 0 0) 0x7f0cf8012a10 con 0x7f0d04099660 2024-09-14T03:40:21.727 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.725+0000 7f0ceb7fe640 1 -- 172.21.15.83:0/3515576781 <== mon.2 v2:172.21.15.83:3302/0 5 ==== osd_map(21..21 src has 1..21) ==== 3027+0+0 (secure 0 0 0) 0x7f0cf8068880 con 0x7f0d04099660 2024-09-14T03:40:21.733 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.732+0000 7f0ceb7fe640 1 -- 172.21.15.83:0/3515576781 <== mon.2 v2:172.21.15.83:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+157506 (secure 0 0 0) 0x7f0cf80676b0 con 0x7f0d04099660 2024-09-14T03:40:21.816 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.814+0000 7f8077273640 1 -- 172.21.15.83:0/3100784342 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 0} v 0) -- 0x7f807006e6b0 con 0x7f80701322e0 2024-09-14T03:40:21.816 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.815+0000 7f8065ffb640 1 -- 172.21.15.83:0/3100784342 <== mon.0 v2:172.21.15.83:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 0}]=0 v0) ==== 74+0+12 (secure 0 0 0) 0x7f8068068710 con 0x7f80701322e0 2024-09-14T03:40:21.816 INFO:teuthology.orchestra.run.smithi083.stdout:38654705681 2024-09-14T03:40:21.818 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.816+0000 7f8077273640 1 -- 172.21.15.83:0/3100784342 >> v1:172.21.15.83:6800/1803897559 conn(0x7f8054051330 legacy=0x7f80540537f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:21.818 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.816+0000 7f8077273640 1 -- 172.21.15.83:0/3100784342 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f80701322e0 msgr2=0x7f8070124b10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:21.819 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.816+0000 7f8077273640 1 --2- 172.21.15.83:0/3100784342 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f80701322e0 0x7f8070124b10 secure :-1 s=READY pgs=38 cs=0 l=1 rev1=1 crypto rx=0x7f806800ec50 tx=0x7f806800c6a0 comp rx=0 tx=0).stop 2024-09-14T03:40:21.819 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.817+0000 7f8077273640 1 -- 172.21.15.83:0/3100784342 shutdown_connections 2024-09-14T03:40:21.819 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.817+0000 7f8077273640 1 --2- 172.21.15.83:0/3100784342 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f80701322e0 0x7f8070124b10 unknown :-1 s=CLOSED pgs=38 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:21.819 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.817+0000 7f8077273640 1 --2- 172.21.15.83:0/3100784342 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f807006edc0 0x7f80701245d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:21.819 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.817+0000 7f8077273640 1 --2- 172.21.15.83:0/3100784342 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f807006e2d0 0x7f8070124090 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:21.819 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.817+0000 7f8077273640 1 -- 172.21.15.83:0/3100784342 >> 172.21.15.83:0/3100784342 conn(0x7f807009e490 msgr2=0x7f8070127380 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:21.820 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.817+0000 7f8077273640 1 -- 172.21.15.83:0/3100784342 shutdown_connections 2024-09-14T03:40:21.820 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.817+0000 7f8077273640 1 -- 172.21.15.83:0/3100784342 wait complete. 2024-09-14T03:40:21.829 INFO:tasks.ceph.ceph_manager.ceph:need seq 38654705682 got 38654705681 for osd.0 2024-09-14T03:40:21.854 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.852+0000 7fd77427a640 1 -- 172.21.15.83:0/618193875 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 1} v 0) -- 0x7fd76c1298f0 con 0x7fd76c12cd60 2024-09-14T03:40:21.854 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.852+0000 7fd762ffd640 1 -- 172.21.15.83:0/618193875 <== mon.2 v2:172.21.15.83:3302/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 1}]=0 v0) ==== 74+0+12 (secure 0 0 0) 0x7fd7640677c0 con 0x7fd76c12cd60 2024-09-14T03:40:21.854 INFO:teuthology.orchestra.run.smithi083.stdout:47244640273 2024-09-14T03:40:21.856 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.854+0000 7fd77427a640 1 -- 172.21.15.83:0/618193875 >> v1:172.21.15.83:6800/1803897559 conn(0x7fd750051330 legacy=0x7fd7500537f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:21.856 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.854+0000 7fd77427a640 1 -- 172.21.15.83:0/618193875 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fd76c12cd60 msgr2=0x7fd76c12fe30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:21.856 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.854+0000 7fd77427a640 1 --2- 172.21.15.83:0/618193875 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fd76c12cd60 0x7fd76c12fe30 secure :-1 s=READY pgs=29 cs=0 l=1 rev1=1 crypto rx=0x7fd76400c790 tx=0x7fd76400cc50 comp rx=0 tx=0).stop 2024-09-14T03:40:21.856 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.854+0000 7fd77427a640 1 -- 172.21.15.83:0/618193875 shutdown_connections 2024-09-14T03:40:21.856 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.854+0000 7fd77427a640 1 --2- 172.21.15.83:0/618193875 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fd76c12cd60 0x7fd76c12fe30 unknown :-1 s=CLOSED pgs=29 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:21.856 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.854+0000 7fd77427a640 1 --2- 172.21.15.83:0/618193875 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fd76c129e30 0x7fd76c1c3350 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:21.856 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.854+0000 7fd77427a640 1 --2- 172.21.15.83:0/618193875 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fd76c129510 0x7fd76c1c2e10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:21.856 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.854+0000 7fd77427a640 1 -- 172.21.15.83:0/618193875 >> 172.21.15.83:0/618193875 conn(0x7fd76c09d370 msgr2=0x7fd76c128190 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:21.856 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.854+0000 7fd77427a640 1 -- 172.21.15.83:0/618193875 shutdown_connections 2024-09-14T03:40:21.857 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.854+0000 7fd77427a640 1 -- 172.21.15.83:0/618193875 wait complete. 2024-09-14T03:40:21.862 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.861+0000 7f2a567a1640 1 -- 172.21.15.83:0/1165836833 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 3} v 0) -- 0x7f2a50115c50 con 0x7f2a50129e30 2024-09-14T03:40:21.862 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.861+0000 7f2a4cff9640 1 -- 172.21.15.83:0/1165836833 <== mon.1 v2:172.21.15.83:3301/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 3}]=0 v0) ==== 74+0+12 (secure 0 0 0) 0x7f2a44088ee0 con 0x7f2a50129e30 2024-09-14T03:40:21.863 INFO:teuthology.orchestra.run.smithi083.stdout:47244640273 2024-09-14T03:40:21.865 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.863+0000 7f2a567a1640 1 -- 172.21.15.83:0/1165836833 >> v1:172.21.15.83:6800/1803897559 conn(0x7f2a3c051300 legacy=0x7f2a3c0537c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:21.865 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.863+0000 7f2a567a1640 1 -- 172.21.15.83:0/1165836833 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f2a50129e30 msgr2=0x7f2a50133110 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:21.865 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.863+0000 7f2a567a1640 1 --2- 172.21.15.83:0/1165836833 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f2a50129e30 0x7f2a50133110 secure :-1 s=READY pgs=29 cs=0 l=1 rev1=1 crypto rx=0x7f2a44000c00 tx=0x7f2a44032f80 comp rx=0 tx=0).stop 2024-09-14T03:40:21.865 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.863+0000 7f2a567a1640 1 -- 172.21.15.83:0/1165836833 shutdown_connections 2024-09-14T03:40:21.865 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.863+0000 7f2a567a1640 1 --2- 172.21.15.83:0/1165836833 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f2a5012cd60 0x7f2a50133650 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:21.865 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.863+0000 7f2a567a1640 1 --2- 172.21.15.83:0/1165836833 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f2a50129e30 0x7f2a50133110 unknown :-1 s=CLOSED pgs=29 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:21.865 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.863+0000 7f2a567a1640 1 --2- 172.21.15.83:0/1165836833 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f2a50129510 0x7f2a50132bd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:21.865 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.863+0000 7f2a567a1640 1 -- 172.21.15.83:0/1165836833 >> 172.21.15.83:0/1165836833 conn(0x7f2a5009d370 msgr2=0x7f2a5009a9f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:21.866 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.863+0000 7f2a567a1640 1 -- 172.21.15.83:0/1165836833 shutdown_connections 2024-09-14T03:40:21.866 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.863+0000 7f2a567a1640 1 -- 172.21.15.83:0/1165836833 wait complete. 2024-09-14T03:40:21.866 INFO:tasks.ceph.ceph_manager.ceph:need seq 47244640274 got 47244640273 for osd.1 2024-09-14T03:40:21.875 INFO:tasks.ceph.ceph_manager.ceph:need seq 47244640274 got 47244640273 for osd.3 2024-09-14T03:40:21.888 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.887+0000 7f0d09366640 1 -- 172.21.15.83:0/3515576781 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 2} v 0) -- 0x7f0d040698c0 con 0x7f0d04099660 2024-09-14T03:40:21.888 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.887+0000 7f0ceb7fe640 1 -- 172.21.15.83:0/3515576781 <== mon.2 v2:172.21.15.83:3302/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 2}]=0 v0) ==== 74+0+12 (secure 0 0 0) 0x7f0cf8041600 con 0x7f0d04099660 2024-09-14T03:40:21.888 INFO:teuthology.orchestra.run.smithi083.stdout:47244640273 2024-09-14T03:40:21.891 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.889+0000 7f0d09366640 1 -- 172.21.15.83:0/3515576781 >> v1:172.21.15.83:6800/1803897559 conn(0x7f0cf0051300 legacy=0x7f0cf00537c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:21.891 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.889+0000 7f0d09366640 1 -- 172.21.15.83:0/3515576781 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f0d04099660 msgr2=0x7f0d041c5290 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:21.891 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.889+0000 7f0d09366640 1 --2- 172.21.15.83:0/3515576781 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f0d04099660 0x7f0d041c5290 secure :-1 s=READY pgs=30 cs=0 l=1 rev1=1 crypto rx=0x7f0cf800c500 tx=0x7f0cf800c9c0 comp rx=0 tx=0).stop 2024-09-14T03:40:21.891 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.889+0000 7f0d09366640 1 -- 172.21.15.83:0/3515576781 shutdown_connections 2024-09-14T03:40:21.891 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.889+0000 7f0d09366640 1 --2- 172.21.15.83:0/3515576781 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f0d0412f1c0 0x7f0d04071320 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:21.891 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.889+0000 7f0d09366640 1 --2- 172.21.15.83:0/3515576781 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f0d04099660 0x7f0d041c5290 unknown :-1 s=CLOSED pgs=30 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:21.891 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.889+0000 7f0d09366640 1 --2- 172.21.15.83:0/3515576781 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f0d04098d40 0x7f0d041c4d50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:21.891 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.889+0000 7f0d09366640 1 -- 172.21.15.83:0/3515576781 >> 172.21.15.83:0/3515576781 conn(0x7f0d040a1790 msgr2=0x7f0d04128440 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:21.891 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.889+0000 7f0d09366640 1 -- 172.21.15.83:0/3515576781 shutdown_connections 2024-09-14T03:40:21.891 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:21.889+0000 7f0d09366640 1 -- 172.21.15.83:0/3515576781 wait complete. 2024-09-14T03:40:21.900 INFO:tasks.ceph.ceph_manager.ceph:need seq 47244640274 got 47244640273 for osd.2 2024-09-14T03:40:22.830 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.0 2024-09-14T03:40:22.868 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.1 2024-09-14T03:40:22.876 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.3 2024-09-14T03:40:22.902 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.2 2024-09-14T03:40:22.941 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:22.939+0000 7fe82103e640 1 -- 172.21.15.83:0/314246284 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fe81c131240 msgr2=0x7fe81c133630 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:22.941 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:22.939+0000 7fe82103e640 1 --2- 172.21.15.83:0/314246284 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fe81c131240 0x7fe81c133630 secure :-1 s=READY pgs=30 cs=0 l=1 rev1=1 crypto rx=0x7fe814009ca0 tx=0x7fe81402a950 comp rx=0 tx=0).stop 2024-09-14T03:40:22.942 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:22.940+0000 7fe82103e640 1 -- 172.21.15.83:0/314246284 shutdown_connections 2024-09-14T03:40:22.942 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:22.940+0000 7fe82103e640 1 --2- 172.21.15.83:0/314246284 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fe81c131240 0x7fe81c133630 unknown :-1 s=CLOSED pgs=30 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:22.942 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:22.940+0000 7fe82103e640 1 --2- 172.21.15.83:0/314246284 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fe81c12e910 0x7fe81c130d00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:22.942 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:22.940+0000 7fe82103e640 1 -- 172.21.15.83:0/314246284 >> 172.21.15.83:0/314246284 conn(0x7fe81c09c230 msgr2=0x7fe81c09c640 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:22.942 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:22.940+0000 7fe82103e640 1 -- 172.21.15.83:0/314246284 shutdown_connections 2024-09-14T03:40:22.942 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:22.940+0000 7fe82103e640 1 -- 172.21.15.83:0/314246284 wait complete. 2024-09-14T03:40:22.945 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:22.941+0000 7fe82103e640 1 Processor -- start 2024-09-14T03:40:22.945 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:22.941+0000 7fe82103e640 1 -- start start 2024-09-14T03:40:22.945 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:22.941+0000 7fe82103e640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fe81c095d40 0x7fe81c12c6f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:22.945 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:22.941+0000 7fe82103e640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fe81c12e910 0x7fe81c12cc30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:22.945 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:22.941+0000 7fe82103e640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fe81c131240 0x7fe81c12d170 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:22.945 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:22.941+0000 7fe82103e640 1 -- --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7fe81c1344f0 con 0x7fe81c131240 2024-09-14T03:40:22.945 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:22.941+0000 7fe82103e640 1 -- --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7fe81c134370 con 0x7fe81c095d40 2024-09-14T03:40:22.945 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:22.941+0000 7fe82103e640 1 -- --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7fe81c134670 con 0x7fe81c12e910 2024-09-14T03:40:22.945 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:22.941+0000 7fe81a59c640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fe81c12e910 0x7fe81c12cc30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:22.945 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:22.941+0000 7fe81a59c640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fe81c12e910 0x7fe81c12cc30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3302/0 says I am v2:172.21.15.83:37756/0 (socket says 172.21.15.83:37756) 2024-09-14T03:40:22.946 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:22.941+0000 7fe81a59c640 1 -- 172.21.15.83:0/2441216432 learned_addr learned my addr 172.21.15.83:0/2441216432 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:40:22.946 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:22.942+0000 7fe81ad9d640 1 --2- 172.21.15.83:0/2441216432 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fe81c095d40 0x7fe81c12c6f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:22.946 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:22.942+0000 7fe81b59e640 1 --2- 172.21.15.83:0/2441216432 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fe81c131240 0x7fe81c12d170 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:22.946 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:22.942+0000 7fe81a59c640 1 -- 172.21.15.83:0/2441216432 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fe81c095d40 msgr2=0x7fe81c12c6f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:22.946 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:22.942+0000 7fe81a59c640 1 --2- 172.21.15.83:0/2441216432 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fe81c095d40 0x7fe81c12c6f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:22.946 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:22.942+0000 7fe81a59c640 1 -- 172.21.15.83:0/2441216432 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fe81c131240 msgr2=0x7fe81c12d170 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:22.946 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:22.942+0000 7fe81a59c640 1 --2- 172.21.15.83:0/2441216432 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fe81c131240 0x7fe81c12d170 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:22.946 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:22.942+0000 7fe81a59c640 1 -- 172.21.15.83:0/2441216432 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fe81c12e050 con 0x7fe81c12e910 2024-09-14T03:40:22.946 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:22.942+0000 7fe81b59e640 1 --2- 172.21.15.83:0/2441216432 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fe81c131240 0x7fe81c12d170 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:40:22.946 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:22.942+0000 7fe81ad9d640 1 --2- 172.21.15.83:0/2441216432 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fe81c095d40 0x7fe81c12c6f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:40:22.946 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:22.942+0000 7fe81a59c640 1 --2- 172.21.15.83:0/2441216432 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fe81c12e910 0x7fe81c12cc30 secure :-1 s=READY pgs=32 cs=0 l=1 rev1=1 crypto rx=0x7fe81000ea60 tx=0x7fe81000ef20 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:40:22.946 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:22.943+0000 7fe8037fe640 1 -- 172.21.15.83:0/2441216432 <== mon.2 v2:172.21.15.83:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe81000cde0 con 0x7fe81c12e910 2024-09-14T03:40:22.946 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:22.943+0000 7fe82103e640 1 -- 172.21.15.83:0/2441216432 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fe81c073f70 con 0x7fe81c12e910 2024-09-14T03:40:22.947 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:22.943+0000 7fe82103e640 1 -- 172.21.15.83:0/2441216432 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7fe81c08c290 con 0x7fe81c12e910 2024-09-14T03:40:22.947 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:22.943+0000 7fe8037fe640 1 -- 172.21.15.83:0/2441216432 <== mon.2 v2:172.21.15.83:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fe810004020 con 0x7fe81c12e910 2024-09-14T03:40:22.947 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:22.943+0000 7fe8037fe640 1 -- 172.21.15.83:0/2441216432 <== mon.2 v2:172.21.15.83:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe810004450 con 0x7fe81c12e910 2024-09-14T03:40:22.949 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:22.944+0000 7fe82103e640 1 -- 172.21.15.83:0/2441216432 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fe81c096120 con 0x7fe81c12e910 2024-09-14T03:40:22.949 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:22.945+0000 7fe8037fe640 1 -- 172.21.15.83:0/2441216432 <== mon.2 v2:172.21.15.83:3302/0 4 ==== mgrmap(e 9) ==== 63442+0+0 (secure 0 0 0) 0x7fe8100035f0 con 0x7fe81c12e910 2024-09-14T03:40:22.950 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:22.946+0000 7fe8037fe640 1 -- 172.21.15.83:0/2441216432 <== mon.2 v2:172.21.15.83:3302/0 5 ==== osd_map(21..21 src has 1..21) ==== 3027+0+0 (secure 0 0 0) 0x7fe810069e20 con 0x7fe81c12e910 2024-09-14T03:40:22.950 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:22.949+0000 7fe8037fe640 1 -- 172.21.15.83:0/2441216432 <== mon.2 v2:172.21.15.83:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+157506 (secure 0 0 0) 0x7fe81c12e050 con 0x7fe81c12e910 2024-09-14T03:40:22.998 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:22.994+0000 7f00ea51a640 1 -- 172.21.15.83:0/119333059 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f00e412bd60 msgr2=0x7f00e412c210 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:22.998 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:22.994+0000 7f00ea51a640 1 --2- 172.21.15.83:0/119333059 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f00e412bd60 0x7f00e412c210 secure :-1 s=READY pgs=45 cs=0 l=1 rev1=1 crypto rx=0x7f00dc009be0 tx=0x7f00dc02a710 comp rx=0 tx=0).stop 2024-09-14T03:40:22.998 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:22.994+0000 7f00ea51a640 1 -- 172.21.15.83:0/119333059 shutdown_connections 2024-09-14T03:40:22.998 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:22.995+0000 7f00ea51a640 1 --2- 172.21.15.83:0/119333059 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f00e4093140 0x7f00e412b820 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:22.998 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:22.995+0000 7f00ea51a640 1 --2- 172.21.15.83:0/119333059 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f00e412bd60 0x7f00e412c210 unknown :-1 s=CLOSED pgs=45 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:22.998 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:22.995+0000 7f00ea51a640 1 --2- 172.21.15.83:0/119333059 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f00e4092820 0x7f00e4092c00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:22.998 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:22.995+0000 7f00ea51a640 1 -- 172.21.15.83:0/119333059 >> 172.21.15.83:0/119333059 conn(0x7f00e409e490 msgr2=0x7f00e409e8a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:22.998 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:22.995+0000 7f00ea51a640 1 -- 172.21.15.83:0/119333059 shutdown_connections 2024-09-14T03:40:22.998 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:22.995+0000 7f00ea51a640 1 -- 172.21.15.83:0/119333059 wait complete. 2024-09-14T03:40:22.999 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:22.995+0000 7f00ea51a640 1 Processor -- start 2024-09-14T03:40:22.999 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:22.996+0000 7f00ea51a640 1 -- start start 2024-09-14T03:40:22.999 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:22.996+0000 7f00ea51a640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f00e4092820 0x7f00e408bf80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:22.999 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:22.996+0000 7f00ea51a640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f00e4093140 0x7f00e408c4c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:22.999 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:22.996+0000 7f00ea51a640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f00e412bd60 0x7f00e408ca00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:22.999 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:22.996+0000 7f00ea51a640 1 -- --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7f00e406eb10 con 0x7f00e412bd60 2024-09-14T03:40:22.999 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:22.996+0000 7f00ea51a640 1 -- --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7f00e406e990 con 0x7f00e4092820 2024-09-14T03:40:22.999 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:22.996+0000 7f00ea51a640 1 -- --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7f00e406ec90 con 0x7f00e4093140 2024-09-14T03:40:22.999 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:22.996+0000 7f00e3fff640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f00e4092820 0x7f00e408bf80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:22.999 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:22.996+0000 7f00e8ab7640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f00e412bd60 0x7f00e408ca00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:22.999 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:22.996+0000 7f00e3fff640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f00e4092820 0x7f00e408bf80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3301/0 says I am v2:172.21.15.83:35670/0 (socket says 172.21.15.83:35670) 2024-09-14T03:40:22.999 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:22.996+0000 7f00e3fff640 1 -- 172.21.15.83:0/2110983447 learned_addr learned my addr 172.21.15.83:0/2110983447 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:40:22.999 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:22.996+0000 7f00e8ab7640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f00e412bd60 0x7f00e408ca00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3300/0 says I am v2:172.21.15.83:39952/0 (socket says 172.21.15.83:39952) 2024-09-14T03:40:22.999 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:22.996+0000 7f00e37fe640 1 --2- 172.21.15.83:0/2110983447 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f00e4093140 0x7f00e408c4c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:23.001 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:22.997+0000 7f00e8ab7640 1 -- 172.21.15.83:0/2110983447 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f00e4093140 msgr2=0x7f00e408c4c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:23.001 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:22.997+0000 7f00e8ab7640 1 --2- 172.21.15.83:0/2110983447 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f00e4093140 0x7f00e408c4c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:23.001 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:22.997+0000 7f00e8ab7640 1 -- 172.21.15.83:0/2110983447 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f00e4092820 msgr2=0x7f00e408bf80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:23.001 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:22.997+0000 7f00e8ab7640 1 --2- 172.21.15.83:0/2110983447 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f00e4092820 0x7f00e408bf80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:23.001 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:22.997+0000 7f00e8ab7640 1 -- 172.21.15.83:0/2110983447 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f00e408d850 con 0x7f00e412bd60 2024-09-14T03:40:23.001 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:22.997+0000 7f00e8ab7640 1 --2- 172.21.15.83:0/2110983447 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f00e412bd60 0x7f00e408ca00 secure :-1 s=READY pgs=46 cs=0 l=1 rev1=1 crypto rx=0x7f00e40765f0 tx=0x7f00dc003e70 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:40:23.001 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:22.997+0000 7f00e0ff9640 1 -- 172.21.15.83:0/2110983447 <== mon.0 v2:172.21.15.83:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f00dc041020 con 0x7f00e412bd60 2024-09-14T03:40:23.001 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:22.997+0000 7f00e0ff9640 1 -- 172.21.15.83:0/2110983447 <== mon.0 v2:172.21.15.83:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f00dc004040 con 0x7f00e412bd60 2024-09-14T03:40:23.001 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:22.997+0000 7f00ea51a640 1 -- 172.21.15.83:0/2110983447 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f00e4073d90 con 0x7f00e412bd60 2024-09-14T03:40:23.002 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:22.997+0000 7f00e0ff9640 1 -- 172.21.15.83:0/2110983447 <== mon.0 v2:172.21.15.83:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f00dc036040 con 0x7f00e412bd60 2024-09-14T03:40:23.002 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:22.997+0000 7f00ea51a640 1 -- 172.21.15.83:0/2110983447 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f00e41c4750 con 0x7f00e412bd60 2024-09-14T03:40:23.002 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:22.999+0000 7f00ca7fc640 1 -- 172.21.15.83:0/2110983447 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f00e412b820 con 0x7f00e412bd60 2024-09-14T03:40:23.002 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:22.999+0000 7f00e0ff9640 1 -- 172.21.15.83:0/2110983447 <== mon.0 v2:172.21.15.83:3300/0 4 ==== mgrmap(e 9) ==== 63442+0+0 (secure 0 0 0) 0x7f00dc044020 con 0x7f00e412bd60 2024-09-14T03:40:23.002 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:22.999+0000 7f00e0ff9640 1 -- 172.21.15.83:0/2110983447 <== mon.0 v2:172.21.15.83:3300/0 5 ==== osd_map(21..21 src has 1..21) ==== 3027+0+0 (secure 0 0 0) 0x7f00dc089d90 con 0x7f00e412bd60 2024-09-14T03:40:23.008 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.006+0000 7f00e0ff9640 1 -- 172.21.15.83:0/2110983447 <== mon.0 v2:172.21.15.83:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+157506 (secure 0 0 0) 0x7f00dc08d030 con 0x7f00e412bd60 2024-09-14T03:40:23.026 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.024+0000 7f00e5951640 1 -- 172.21.15.83:0/218056934 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f00e0099660 msgr2=0x7f00e012eab0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:23.026 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.024+0000 7f00e5951640 1 --2- 172.21.15.83:0/218056934 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f00e0099660 0x7f00e012eab0 secure :-1 s=READY pgs=31 cs=0 l=1 rev1=1 crypto rx=0x7f00d4009830 tx=0x7f00d402a360 comp rx=0 tx=0).stop 2024-09-14T03:40:23.026 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.024+0000 7f00e5951640 1 -- 172.21.15.83:0/218056934 shutdown_connections 2024-09-14T03:40:23.026 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.024+0000 7f00e5951640 1 --2- 172.21.15.83:0/218056934 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f00e0099660 0x7f00e012eab0 unknown :-1 s=CLOSED pgs=31 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:23.026 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.024+0000 7f00e5951640 1 --2- 172.21.15.83:0/218056934 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f00e012eff0 0x7f00e012f480 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:23.026 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.024+0000 7f00e5951640 1 --2- 172.21.15.83:0/218056934 >> v2:172.21.15.83:3300/0 conn(0x7f00e0098d40 0x7f00e0099120 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:23.026 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.024+0000 7f00e5951640 1 -- 172.21.15.83:0/218056934 >> 172.21.15.83:0/218056934 conn(0x7f00e00a1790 msgr2=0x7f00e00a1ba0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:23.027 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.024+0000 7f00e5951640 1 -- 172.21.15.83:0/218056934 shutdown_connections 2024-09-14T03:40:23.027 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.024+0000 7f00e5951640 1 -- 172.21.15.83:0/218056934 wait complete. 2024-09-14T03:40:23.029 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.025+0000 7f00e5951640 1 Processor -- start 2024-09-14T03:40:23.029 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.025+0000 7f00e5951640 1 -- start start 2024-09-14T03:40:23.029 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.026+0000 7f00e5951640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f00e0098d40 0x7f00e01c4cb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:23.029 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.026+0000 7f00e5951640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f00e0099660 0x7f00e01c51f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:23.029 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.026+0000 7f00e5951640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f00e012eff0 0x7f00e0132910 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:23.030 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.026+0000 7f00e5951640 1 -- --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7f00e01365b0 con 0x7f00e0098d40 2024-09-14T03:40:23.030 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.026+0000 7f00e5951640 1 -- --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7f00e0136430 con 0x7f00e0099660 2024-09-14T03:40:23.030 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.026+0000 7f00e5951640 1 -- --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7f00e0136730 con 0x7f00e012eff0 2024-09-14T03:40:23.030 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.026+0000 7f00de7fc640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f00e0099660 0x7f00e01c51f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:23.030 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.026+0000 7f00de7fc640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f00e0099660 0x7f00e01c51f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3301/0 says I am v2:172.21.15.83:35688/0 (socket says 172.21.15.83:35688) 2024-09-14T03:40:23.030 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.026+0000 7f00de7fc640 1 -- 172.21.15.83:0/2397291349 learned_addr learned my addr 172.21.15.83:0/2397291349 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:40:23.030 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.026+0000 7f00deffd640 1 --2- 172.21.15.83:0/2397291349 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f00e0098d40 0x7f00e01c4cb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:23.030 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.026+0000 7f00df7fe640 1 --2- 172.21.15.83:0/2397291349 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f00e012eff0 0x7f00e0132910 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:23.030 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.026+0000 7f00de7fc640 1 -- 172.21.15.83:0/2397291349 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f00e012eff0 msgr2=0x7f00e0132910 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:23.030 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.026+0000 7f00de7fc640 1 --2- 172.21.15.83:0/2397291349 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f00e012eff0 0x7f00e0132910 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:23.030 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.026+0000 7f00de7fc640 1 -- 172.21.15.83:0/2397291349 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f00e0098d40 msgr2=0x7f00e01c4cb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:23.030 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.026+0000 7f00de7fc640 1 --2- 172.21.15.83:0/2397291349 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f00e0098d40 0x7f00e01c4cb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:23.030 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.026+0000 7f00de7fc640 1 -- 172.21.15.83:0/2397291349 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f00e006e910 con 0x7f00e0099660 2024-09-14T03:40:23.030 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.026+0000 7f00df7fe640 1 --2- 172.21.15.83:0/2397291349 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f00e012eff0 0x7f00e0132910 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-09-14T03:40:23.031 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.026+0000 7f00de7fc640 1 --2- 172.21.15.83:0/2397291349 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f00e0099660 0x7f00e01c51f0 secure :-1 s=READY pgs=32 cs=0 l=1 rev1=1 crypto rx=0x7f00d4009780 tx=0x7f00d4033b70 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:40:23.031 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.027+0000 7f00c7fff640 1 -- 172.21.15.83:0/2397291349 <== mon.1 v2:172.21.15.83:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f00d403f020 con 0x7f00e0099660 2024-09-14T03:40:23.031 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.027+0000 7f00e5951640 1 -- 172.21.15.83:0/2397291349 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f00e00744b0 con 0x7f00e0099660 2024-09-14T03:40:23.031 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.027+0000 7f00e5951640 1 -- 172.21.15.83:0/2397291349 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7f00e01c5730 con 0x7f00e0099660 2024-09-14T03:40:23.031 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.027+0000 7f00c7fff640 1 -- 172.21.15.83:0/2397291349 <== mon.1 v2:172.21.15.83:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f00d40044c0 con 0x7f00e0099660 2024-09-14T03:40:23.031 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.027+0000 7f00c7fff640 1 -- 172.21.15.83:0/2397291349 <== mon.1 v2:172.21.15.83:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f00d4037040 con 0x7f00e0099660 2024-09-14T03:40:23.031 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.028+0000 7f00c7fff640 1 -- 172.21.15.83:0/2397291349 <== mon.1 v2:172.21.15.83:3301/0 4 ==== mgrmap(e 9) ==== 63442+0+0 (secure 0 0 0) 0x7f00d4042020 con 0x7f00e0099660 2024-09-14T03:40:23.031 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.028+0000 7f00e5951640 1 -- 172.21.15.83:0/2397291349 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f00c0005290 con 0x7f00e0099660 2024-09-14T03:40:23.031 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.028+0000 7f00c7fff640 1 -- 172.21.15.83:0/2397291349 <== mon.1 v2:172.21.15.83:3301/0 5 ==== osd_map(21..21 src has 1..21) ==== 3027+0+0 (secure 0 0 0) 0x7f00d4088960 con 0x7f00e0099660 2024-09-14T03:40:23.033 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.032+0000 7f00c7fff640 1 -- 172.21.15.83:0/2397291349 <== mon.1 v2:172.21.15.83:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+157506 (secure 0 0 0) 0x7f00d4049030 con 0x7f00e0099660 2024-09-14T03:40:23.038 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.036+0000 7f194bc0f640 1 -- 172.21.15.83:0/2570878036 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f1944098d40 msgr2=0x7f1944099120 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:23.038 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.036+0000 7f194bc0f640 1 --2- 172.21.15.83:0/2570878036 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f1944098d40 0x7f1944099120 secure :-1 s=READY pgs=47 cs=0 l=1 rev1=1 crypto rx=0x7f1940005fd0 tx=0x7f194002a410 comp rx=0 tx=0).stop 2024-09-14T03:40:23.038 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.036+0000 7f194bc0f640 1 -- 172.21.15.83:0/2570878036 shutdown_connections 2024-09-14T03:40:23.038 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.036+0000 7f194bc0f640 1 --2- 172.21.15.83:0/2570878036 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f194412eff0 0x7f194412f480 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:23.038 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.036+0000 7f194bc0f640 1 --2- 172.21.15.83:0/2570878036 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f1944099660 0x7f194412eab0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:23.038 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.036+0000 7f194bc0f640 1 --2- 172.21.15.83:0/2570878036 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f1944098d40 0x7f1944099120 unknown :-1 s=CLOSED pgs=47 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:23.038 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.036+0000 7f194bc0f640 1 -- 172.21.15.83:0/2570878036 >> 172.21.15.83:0/2570878036 conn(0x7f19440a1790 msgr2=0x7f19440a1ba0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:23.038 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.036+0000 7f194bc0f640 1 -- 172.21.15.83:0/2570878036 shutdown_connections 2024-09-14T03:40:23.039 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.036+0000 7f194bc0f640 1 -- 172.21.15.83:0/2570878036 wait complete. 2024-09-14T03:40:23.040 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.037+0000 7f194bc0f640 1 Processor -- start 2024-09-14T03:40:23.040 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.037+0000 7f194bc0f640 1 -- start start 2024-09-14T03:40:23.040 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.038+0000 7f194bc0f640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f1944098d40 0x7f19441c4a00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:23.040 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.038+0000 7f194bc0f640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f1944099660 0x7f19441c4f40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:23.040 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.038+0000 7f194bc0f640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f194412eff0 0x7f19441c5480 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:23.041 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.038+0000 7f194bc0f640 1 -- --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7f1944136570 con 0x7f194412eff0 2024-09-14T03:40:23.041 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.038+0000 7f194bc0f640 1 -- --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7f19441363f0 con 0x7f1944098d40 2024-09-14T03:40:23.041 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.038+0000 7f194bc0f640 1 -- --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7f19441366f0 con 0x7f1944099660 2024-09-14T03:40:23.041 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.038+0000 7f194a1ac640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f194412eff0 0x7f19441c5480 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:23.041 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.038+0000 7f194a1ac640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f194412eff0 0x7f19441c5480 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3300/0 says I am v2:172.21.15.83:39986/0 (socket says 172.21.15.83:39986) 2024-09-14T03:40:23.041 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.038+0000 7f194a1ac640 1 -- 172.21.15.83:0/964431192 learned_addr learned my addr 172.21.15.83:0/964431192 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:40:23.041 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.039+0000 7f194a1ac640 1 -- 172.21.15.83:0/964431192 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f1944099660 msgr2=0x7f19441c4f40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:23.041 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.039+0000 7f194a1ac640 1 --2- 172.21.15.83:0/964431192 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f1944099660 0x7f19441c4f40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:23.041 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.039+0000 7f194a1ac640 1 -- 172.21.15.83:0/964431192 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f1944098d40 msgr2=0x7f19441c4a00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:23.041 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.039+0000 7f194a1ac640 1 --2- 172.21.15.83:0/964431192 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f1944098d40 0x7f19441c4a00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:23.041 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.039+0000 7f194a1ac640 1 -- 172.21.15.83:0/964431192 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f1944129460 con 0x7f194412eff0 2024-09-14T03:40:23.041 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.039+0000 7f194a1ac640 1 --2- 172.21.15.83:0/964431192 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f194412eff0 0x7f19441c5480 secure :-1 s=READY pgs=48 cs=0 l=1 rev1=1 crypto rx=0x7f193c00efd0 tx=0x7f193c00c6a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:40:23.043 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.042+0000 7f193a7fc640 1 -- 172.21.15.83:0/964431192 <== mon.0 v2:172.21.15.83:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f193c003b70 con 0x7f194412eff0 2024-09-14T03:40:23.043 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.042+0000 7f193a7fc640 1 -- 172.21.15.83:0/964431192 <== mon.0 v2:172.21.15.83:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f193c003e00 con 0x7f194412eff0 2024-09-14T03:40:23.043 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.042+0000 7f193a7fc640 1 -- 172.21.15.83:0/964431192 <== mon.0 v2:172.21.15.83:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f193c004280 con 0x7f194412eff0 2024-09-14T03:40:23.043 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.042+0000 7f194bc0f640 1 -- 172.21.15.83:0/964431192 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f194406ea40 con 0x7f194412eff0 2024-09-14T03:40:23.044 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.043+0000 7f194bc0f640 1 -- 172.21.15.83:0/964431192 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f1944132b00 con 0x7f194412eff0 2024-09-14T03:40:23.045 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.043+0000 7f193a7fc640 1 -- 172.21.15.83:0/964431192 <== mon.0 v2:172.21.15.83:3300/0 4 ==== mgrmap(e 9) ==== 63442+0+0 (secure 0 0 0) 0x7f193c004420 con 0x7f194412eff0 2024-09-14T03:40:23.045 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.044+0000 7f193a7fc640 1 -- 172.21.15.83:0/964431192 <== mon.0 v2:172.21.15.83:3300/0 5 ==== osd_map(21..21 src has 1..21) ==== 3027+0+0 (secure 0 0 0) 0x7f193c06edb0 con 0x7f194412eff0 2024-09-14T03:40:23.045 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.044+0000 7f194bc0f640 1 -- 172.21.15.83:0/964431192 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f1944099120 con 0x7f194412eff0 2024-09-14T03:40:23.055 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.054+0000 7f193a7fc640 1 -- 172.21.15.83:0/964431192 <== mon.0 v2:172.21.15.83:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+157506 (secure 0 0 0) 0x7f193c004710 con 0x7f194412eff0 2024-09-14T03:40:23.149 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.147+0000 7fe82103e640 1 -- 172.21.15.83:0/2441216432 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 0} v 0) -- 0x7fe81c069a30 con 0x7fe81c12e910 2024-09-14T03:40:23.149 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.148+0000 7fe8037fe640 1 -- 172.21.15.83:0/2441216432 <== mon.2 v2:172.21.15.83:3302/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 0}]=0 v0) ==== 74+0+12 (secure 0 0 0) 0x7fe8100689e0 con 0x7fe81c12e910 2024-09-14T03:40:23.149 INFO:teuthology.orchestra.run.smithi083.stdout:38654705683 2024-09-14T03:40:23.152 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.150+0000 7fe82103e640 1 -- 172.21.15.83:0/2441216432 >> v1:172.21.15.83:6800/1803897559 conn(0x7fe808051420 legacy=0x7fe8080538e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:23.152 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.150+0000 7fe82103e640 1 -- 172.21.15.83:0/2441216432 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fe81c12e910 msgr2=0x7fe81c12cc30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:23.152 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.150+0000 7fe82103e640 1 --2- 172.21.15.83:0/2441216432 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fe81c12e910 0x7fe81c12cc30 secure :-1 s=READY pgs=32 cs=0 l=1 rev1=1 crypto rx=0x7fe81000ea60 tx=0x7fe81000ef20 comp rx=0 tx=0).stop 2024-09-14T03:40:23.152 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.150+0000 7fe82103e640 1 -- 172.21.15.83:0/2441216432 shutdown_connections 2024-09-14T03:40:23.152 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.150+0000 7fe82103e640 1 --2- 172.21.15.83:0/2441216432 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fe81c131240 0x7fe81c12d170 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:23.152 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.150+0000 7fe82103e640 1 --2- 172.21.15.83:0/2441216432 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fe81c12e910 0x7fe81c12cc30 unknown :-1 s=CLOSED pgs=32 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:23.152 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.150+0000 7fe82103e640 1 --2- 172.21.15.83:0/2441216432 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fe81c095d40 0x7fe81c12c6f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:23.152 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.150+0000 7fe82103e640 1 -- 172.21.15.83:0/2441216432 >> 172.21.15.83:0/2441216432 conn(0x7fe81c09c230 msgr2=0x7fe81c130990 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:23.152 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.150+0000 7fe82103e640 1 -- 172.21.15.83:0/2441216432 shutdown_connections 2024-09-14T03:40:23.153 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.150+0000 7fe82103e640 1 -- 172.21.15.83:0/2441216432 wait complete. 2024-09-14T03:40:23.167 INFO:tasks.ceph.ceph_manager.ceph:need seq 38654705682 got 38654705683 for osd.0 2024-09-14T03:40:23.167 DEBUG:teuthology.parallel:result is None 2024-09-14T03:40:23.196 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.195+0000 7f00e5951640 1 -- 172.21.15.83:0/2397291349 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 3} v 0) -- 0x7f00c0005580 con 0x7f00e0099660 2024-09-14T03:40:23.197 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.195+0000 7f00c7fff640 1 -- 172.21.15.83:0/2397291349 <== mon.1 v2:172.21.15.83:3301/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 3}]=0 v0) ==== 74+0+12 (secure 0 0 0) 0x7f00e006e910 con 0x7f00e0099660 2024-09-14T03:40:23.197 INFO:teuthology.orchestra.run.smithi083.stdout:47244640275 2024-09-14T03:40:23.199 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.197+0000 7f00e5951640 1 -- 172.21.15.83:0/2397291349 >> v1:172.21.15.83:6800/1803897559 conn(0x7f00cc051450 legacy=0x7f00cc053910 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:23.199 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.197+0000 7f00e5951640 1 -- 172.21.15.83:0/2397291349 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f00e0099660 msgr2=0x7f00e01c51f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:23.199 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.197+0000 7f00e5951640 1 --2- 172.21.15.83:0/2397291349 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f00e0099660 0x7f00e01c51f0 secure :-1 s=READY pgs=32 cs=0 l=1 rev1=1 crypto rx=0x7f00d4009780 tx=0x7f00d4033b70 comp rx=0 tx=0).stop 2024-09-14T03:40:23.199 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.197+0000 7f00e5951640 1 -- 172.21.15.83:0/2397291349 shutdown_connections 2024-09-14T03:40:23.199 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.197+0000 7f00e5951640 1 --2- 172.21.15.83:0/2397291349 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f00e012eff0 0x7f00e0132910 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:23.199 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.197+0000 7f00e5951640 1 --2- 172.21.15.83:0/2397291349 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f00e0099660 0x7f00e01c51f0 unknown :-1 s=CLOSED pgs=32 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:23.199 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.197+0000 7f00e5951640 1 --2- 172.21.15.83:0/2397291349 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f00e0098d40 0x7f00e01c4cb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:23.199 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.197+0000 7f00e5951640 1 -- 172.21.15.83:0/2397291349 >> 172.21.15.83:0/2397291349 conn(0x7f00e00a1790 msgr2=0x7f00e0128440 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:23.199 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.197+0000 7f00e5951640 1 -- 172.21.15.83:0/2397291349 shutdown_connections 2024-09-14T03:40:23.199 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.197+0000 7f00e5951640 1 -- 172.21.15.83:0/2397291349 wait complete. 2024-09-14T03:40:23.209 INFO:tasks.ceph.ceph_manager.ceph:need seq 47244640274 got 47244640275 for osd.3 2024-09-14T03:40:23.209 DEBUG:teuthology.parallel:result is None 2024-09-14T03:40:23.210 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.209+0000 7f00ca7fc640 1 -- 172.21.15.83:0/2110983447 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 1} v 0) -- 0x7f00e40008d0 con 0x7f00e412bd60 2024-09-14T03:40:23.211 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.209+0000 7f00e0ff9640 1 -- 172.21.15.83:0/2110983447 <== mon.0 v2:172.21.15.83:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 1}]=0 v0) ==== 74+0+12 (secure 0 0 0) 0x7f00dc04b030 con 0x7f00e412bd60 2024-09-14T03:40:23.211 INFO:teuthology.orchestra.run.smithi083.stdout:47244640275 2024-09-14T03:40:23.211 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.210+0000 7f194bc0f640 1 -- 172.21.15.83:0/964431192 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 2} v 0) -- 0x7f19440698c0 con 0x7f194412eff0 2024-09-14T03:40:23.211 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.210+0000 7f193a7fc640 1 -- 172.21.15.83:0/964431192 <== mon.0 v2:172.21.15.83:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 2}]=0 v0) ==== 74+0+12 (secure 0 0 0) 0x7f193c067c10 con 0x7f194412eff0 2024-09-14T03:40:23.212 INFO:teuthology.orchestra.run.smithi083.stdout:47244640275 2024-09-14T03:40:23.213 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.211+0000 7f00ca7fc640 1 -- 172.21.15.83:0/2110983447 >> v1:172.21.15.83:6800/1803897559 conn(0x7f00d0051420 legacy=0x7f00d00538e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:23.213 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.211+0000 7f00ca7fc640 1 -- 172.21.15.83:0/2110983447 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f00e412bd60 msgr2=0x7f00e408ca00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:23.213 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.211+0000 7f00ca7fc640 1 --2- 172.21.15.83:0/2110983447 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f00e412bd60 0x7f00e408ca00 secure :-1 s=READY pgs=46 cs=0 l=1 rev1=1 crypto rx=0x7f00e40765f0 tx=0x7f00dc003e70 comp rx=0 tx=0).stop 2024-09-14T03:40:23.213 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.211+0000 7f00ca7fc640 1 -- 172.21.15.83:0/2110983447 shutdown_connections 2024-09-14T03:40:23.213 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.211+0000 7f00ca7fc640 1 --2- 172.21.15.83:0/2110983447 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f00e412bd60 0x7f00e408ca00 unknown :-1 s=CLOSED pgs=46 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:23.213 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.211+0000 7f00ca7fc640 1 --2- 172.21.15.83:0/2110983447 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f00e4093140 0x7f00e408c4c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:23.213 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.211+0000 7f00ca7fc640 1 --2- 172.21.15.83:0/2110983447 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f00e4092820 0x7f00e408bf80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:23.213 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.211+0000 7f00ca7fc640 1 -- 172.21.15.83:0/2110983447 >> 172.21.15.83:0/2110983447 conn(0x7f00e409e490 msgr2=0x7f00e4099290 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:23.213 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.211+0000 7f00ca7fc640 1 -- 172.21.15.83:0/2110983447 shutdown_connections 2024-09-14T03:40:23.213 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.211+0000 7f00ca7fc640 1 -- 172.21.15.83:0/2110983447 wait complete. 2024-09-14T03:40:23.214 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.212+0000 7f194bc0f640 1 -- 172.21.15.83:0/964431192 >> v1:172.21.15.83:6800/1803897559 conn(0x7f1928051470 legacy=0x7f1928053930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:23.214 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.212+0000 7f194bc0f640 1 -- 172.21.15.83:0/964431192 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f194412eff0 msgr2=0x7f19441c5480 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:23.214 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.212+0000 7f194bc0f640 1 --2- 172.21.15.83:0/964431192 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f194412eff0 0x7f19441c5480 secure :-1 s=READY pgs=48 cs=0 l=1 rev1=1 crypto rx=0x7f193c00efd0 tx=0x7f193c00c6a0 comp rx=0 tx=0).stop 2024-09-14T03:40:23.214 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.212+0000 7f194bc0f640 1 -- 172.21.15.83:0/964431192 shutdown_connections 2024-09-14T03:40:23.215 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.212+0000 7f194bc0f640 1 --2- 172.21.15.83:0/964431192 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f194412eff0 0x7f19441c5480 unknown :-1 s=CLOSED pgs=48 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:23.215 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.212+0000 7f194bc0f640 1 --2- 172.21.15.83:0/964431192 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f1944099660 0x7f19441c4f40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:23.215 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.212+0000 7f194bc0f640 1 --2- 172.21.15.83:0/964431192 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f1944098d40 0x7f19441c4a00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:23.215 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.212+0000 7f194bc0f640 1 -- 172.21.15.83:0/964431192 >> 172.21.15.83:0/964431192 conn(0x7f19440a1790 msgr2=0x7f1944128440 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:23.215 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.212+0000 7f194bc0f640 1 -- 172.21.15.83:0/964431192 shutdown_connections 2024-09-14T03:40:23.215 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.212+0000 7f194bc0f640 1 -- 172.21.15.83:0/964431192 wait complete. 2024-09-14T03:40:23.224 INFO:tasks.ceph.ceph_manager.ceph:need seq 47244640274 got 47244640275 for osd.1 2024-09-14T03:40:23.225 DEBUG:teuthology.parallel:result is None 2024-09-14T03:40:23.225 INFO:tasks.ceph.ceph_manager.ceph:need seq 47244640274 got 47244640275 for osd.2 2024-09-14T03:40:23.225 DEBUG:teuthology.parallel:result is None 2024-09-14T03:40:23.225 INFO:tasks.ceph.ceph_manager.ceph:waiting for clean 2024-09-14T03:40:23.225 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-09-14T03:40:23.364 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.362+0000 7fa6670cb640 1 -- 172.21.15.83:0/3232574749 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fa660095c20 msgr2=0x7fa660096080 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:23.365 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.362+0000 7fa6670cb640 1 --2- 172.21.15.83:0/3232574749 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fa660095c20 0x7fa660096080 secure :-1 s=READY pgs=33 cs=0 l=1 rev1=1 crypto rx=0x7fa658005fd0 tx=0x7fa65802a6e0 comp rx=0 tx=0).stop 2024-09-14T03:40:23.365 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.362+0000 7fa6670cb640 1 -- 172.21.15.83:0/3232574749 shutdown_connections 2024-09-14T03:40:23.365 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.362+0000 7fa6670cb640 1 --2- 172.21.15.83:0/3232574749 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fa660095c20 0x7fa660096080 unknown :-1 s=CLOSED pgs=33 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:23.365 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.362+0000 7fa6670cb640 1 --2- 172.21.15.83:0/3232574749 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fa66012c160 0x7fa6600956e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:23.365 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.362+0000 7fa6670cb640 1 --2- 172.21.15.83:0/3232574749 >> v2:172.21.15.83:3300/0 conn(0x7fa6600991f0 0x7fa66012bc20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:23.365 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.362+0000 7fa6670cb640 1 -- 172.21.15.83:0/3232574749 >> 172.21.15.83:0/3232574749 conn(0x7fa66009a2e0 msgr2=0x7fa660097ad0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:23.365 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.362+0000 7fa6670cb640 1 -- 172.21.15.83:0/3232574749 shutdown_connections 2024-09-14T03:40:23.365 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.363+0000 7fa6670cb640 1 -- 172.21.15.83:0/3232574749 wait complete. 2024-09-14T03:40:23.365 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.363+0000 7fa6670cb640 1 Processor -- start 2024-09-14T03:40:23.365 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.363+0000 7fa6670cb640 1 -- start start 2024-09-14T03:40:23.367 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.364+0000 7fa6670cb640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fa660095c20 0x7fa660072a40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:23.367 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.364+0000 7fa6670cb640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fa6600991f0 0x7fa660072f80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:23.368 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.364+0000 7fa6670cb640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fa66012c160 0x7fa6600734c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:23.368 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.364+0000 7fa6670cb640 1 -- --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7fa660133530 con 0x7fa6600991f0 2024-09-14T03:40:23.368 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.364+0000 7fa6670cb640 1 -- --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7fa6601333b0 con 0x7fa66012c160 2024-09-14T03:40:23.368 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.364+0000 7fa6670cb640 1 -- --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7fa6601336b0 con 0x7fa660095c20 2024-09-14T03:40:23.368 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.364+0000 7fa664e67640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fa660095c20 0x7fa660072a40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:23.368 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.364+0000 7fa65ffff640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fa6600991f0 0x7fa660072f80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:23.368 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.364+0000 7fa664e67640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fa660095c20 0x7fa660072a40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3302/0 says I am v2:172.21.15.83:37832/0 (socket says 172.21.15.83:37832) 2024-09-14T03:40:23.368 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.364+0000 7fa664e67640 1 -- 172.21.15.83:0/1439694079 learned_addr learned my addr 172.21.15.83:0/1439694079 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:40:23.368 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.364+0000 7fa665668640 1 --2- 172.21.15.83:0/1439694079 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fa66012c160 0x7fa6600734c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:23.368 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.364+0000 7fa664e67640 1 -- 172.21.15.83:0/1439694079 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fa66012c160 msgr2=0x7fa6600734c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:23.368 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.364+0000 7fa664e67640 1 --2- 172.21.15.83:0/1439694079 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fa66012c160 0x7fa6600734c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:23.368 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.364+0000 7fa664e67640 1 -- 172.21.15.83:0/1439694079 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fa6600991f0 msgr2=0x7fa660072f80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:23.368 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.364+0000 7fa664e67640 1 --2- 172.21.15.83:0/1439694079 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fa6600991f0 0x7fa660072f80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:23.369 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.364+0000 7fa664e67640 1 -- 172.21.15.83:0/1439694079 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fa660094840 con 0x7fa660095c20 2024-09-14T03:40:23.369 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.364+0000 7fa665668640 1 --2- 172.21.15.83:0/1439694079 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fa66012c160 0x7fa6600734c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:40:23.369 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.364+0000 7fa65ffff640 1 --2- 172.21.15.83:0/1439694079 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fa6600991f0 0x7fa660072f80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:40:23.369 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.364+0000 7fa664e67640 1 --2- 172.21.15.83:0/1439694079 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fa660095c20 0x7fa660072a40 secure :-1 s=READY pgs=33 cs=0 l=1 rev1=1 crypto rx=0x7fa65000e9f0 tx=0x7fa65000edf0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:40:23.369 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.365+0000 7fa65d7fa640 1 -- 172.21.15.83:0/1439694079 <== mon.2 v2:172.21.15.83:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa65000ccc0 con 0x7fa660095c20 2024-09-14T03:40:23.369 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.365+0000 7fa65d7fa640 1 -- 172.21.15.83:0/1439694079 <== mon.2 v2:172.21.15.83:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fa650010430 con 0x7fa660095c20 2024-09-14T03:40:23.369 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.365+0000 7fa65d7fa640 1 -- 172.21.15.83:0/1439694079 <== mon.2 v2:172.21.15.83:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa650010860 con 0x7fa660095c20 2024-09-14T03:40:23.369 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.365+0000 7fa6670cb640 1 -- 172.21.15.83:0/1439694079 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fa660074020 con 0x7fa660095c20 2024-09-14T03:40:23.369 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.365+0000 7fa6670cb640 1 -- 172.21.15.83:0/1439694079 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7fa66012eb30 con 0x7fa660095c20 2024-09-14T03:40:23.369 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.366+0000 7fa6670cb640 1 -- 172.21.15.83:0/1439694079 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fa66012bc20 con 0x7fa660095c20 2024-09-14T03:40:23.369 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.366+0000 7fa65d7fa640 1 -- 172.21.15.83:0/1439694079 <== mon.2 v2:172.21.15.83:3302/0 4 ==== mgrmap(e 9) ==== 63442+0+0 (secure 0 0 0) 0x7fa650010a00 con 0x7fa660095c20 2024-09-14T03:40:23.369 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.366+0000 7fa65d7fa640 1 -- 172.21.15.83:0/1439694079 <== mon.2 v2:172.21.15.83:3302/0 5 ==== osd_map(21..21 src has 1..21) ==== 3027+0+0 (secure 0 0 0) 0x7fa650069a40 con 0x7fa660095c20 2024-09-14T03:40:23.371 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.370+0000 7fa65d7fa640 1 -- 172.21.15.83:0/1439694079 <== mon.2 v2:172.21.15.83:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+157506 (secure 0 0 0) 0x7fa650010cf0 con 0x7fa660095c20 2024-09-14T03:40:23.515 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.514+0000 7fa6670cb640 1 -- 172.21.15.83:0/1439694079 --> v1:172.21.15.83:6800/1803897559 -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) -- 0x7fa6600699c0 con 0x7fa64c051420 2024-09-14T03:40:23.516 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.514+0000 7fa65d7fa640 1 -- 172.21.15.83:0/1439694079 <== mgr.14106 v1:172.21.15.83:6800/1803897559 1 ==== mgr_command_reply(tid 0: 0 dumped all) ==== 18+0+32011 (unknown 2965378022 0 2510186030) 0x7fa6600699c0 con 0x7fa64c051420 2024-09-14T03:40:23.516 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:40:23.517 INFO:teuthology.orchestra.run.smithi083.stderr:dumped all 2024-09-14T03:40:23.518 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.516+0000 7fa6670cb640 1 -- 172.21.15.83:0/1439694079 >> v1:172.21.15.83:6800/1803897559 conn(0x7fa64c051420 legacy=0x7fa64c0538e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:23.518 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.516+0000 7fa6670cb640 1 -- 172.21.15.83:0/1439694079 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fa660095c20 msgr2=0x7fa660072a40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:23.518 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.516+0000 7fa6670cb640 1 --2- 172.21.15.83:0/1439694079 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fa660095c20 0x7fa660072a40 secure :-1 s=READY pgs=33 cs=0 l=1 rev1=1 crypto rx=0x7fa65000e9f0 tx=0x7fa65000edf0 comp rx=0 tx=0).stop 2024-09-14T03:40:23.518 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.516+0000 7fa6670cb640 1 -- 172.21.15.83:0/1439694079 shutdown_connections 2024-09-14T03:40:23.518 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.516+0000 7fa6670cb640 1 --2- 172.21.15.83:0/1439694079 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fa66012c160 0x7fa6600734c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:23.519 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.516+0000 7fa6670cb640 1 --2- 172.21.15.83:0/1439694079 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fa6600991f0 0x7fa660072f80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:23.519 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.516+0000 7fa6670cb640 1 --2- 172.21.15.83:0/1439694079 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fa660095c20 0x7fa660072a40 unknown :-1 s=CLOSED pgs=33 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:23.519 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.516+0000 7fa6670cb640 1 -- 172.21.15.83:0/1439694079 >> 172.21.15.83:0/1439694079 conn(0x7fa66009a2e0 msgr2=0x7fa66008bf80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:23.519 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.517+0000 7fa6670cb640 1 -- 172.21.15.83:0/1439694079 shutdown_connections 2024-09-14T03:40:23.519 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.517+0000 7fa6670cb640 1 -- 172.21.15.83:0/1439694079 wait complete. 2024-09-14T03:40:23.528 INFO:teuthology.orchestra.run.smithi083.stdout:{"pg_ready":true,"pg_map":{"version":4,"stamp":"2024-09-14T03:40:22.039693+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":139,"num_write_kb":1896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":100,"ondisk_log_size":100,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":4,"num_per_pool_osds":4,"num_per_pool_omap_osds":4,"kb":377487360,"kb_used":109976,"kb_used_data":1864,"kb_used_omap":6,"kb_used_meta":108025,"kb_avail":377377384,"statfs":{"total":386547056640,"available":386434441216,"internally_reserved":0,"allocated":1908736,"data_stored":1388116,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6378,"internal_metadata":110618390},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"0.997195"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":25,"reported_epoch":21,"state":"active+clean","last_fresh":"2024-09-14T03:40:20.036336+0000","last_change":"2024-09-14T03:39:03.811650+0000","last_active":"2024-09-14T03:40:20.036336+0000","last_peered":"2024-09-14T03:40:20.036336+0000","last_clean":"2024-09-14T03:40:20.036336+0000","last_became_active":"2024-09-14T03:39:01.978698+0000","last_became_peered":"2024-09-14T03:39:01.978698+0000","last_unstale":"2024-09-14T03:40:20.036336+0000","last_undegraded":"2024-09-14T03:40:20.036336+0000","last_fullsized":"2024-09-14T03:40:20.036336+0000","mapping_epoch":15,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":16,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_clean_scrub_stamp":"2024-09-14T03:39:00.791161+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:14:21.446815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00098140899999999993,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":25,"reported_epoch":21,"state":"active+clean","last_fresh":"2024-09-14T03:40:20.062773+0000","last_change":"2024-09-14T03:39:03.819819+0000","last_active":"2024-09-14T03:40:20.062773+0000","last_peered":"2024-09-14T03:40:20.062773+0000","last_clean":"2024-09-14T03:40:20.062773+0000","last_became_active":"2024-09-14T03:39:01.978827+0000","last_became_peered":"2024-09-14T03:39:01.978827+0000","last_unstale":"2024-09-14T03:40:20.062773+0000","last_undegraded":"2024-09-14T03:40:20.062773+0000","last_fullsized":"2024-09-14T03:40:20.062773+0000","mapping_epoch":15,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":16,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_clean_scrub_stamp":"2024-09-14T03:39:00.791161+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:40:25.557607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00045411099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":25,"reported_epoch":21,"state":"active+clean","last_fresh":"2024-09-14T03:40:20.036263+0000","last_change":"2024-09-14T03:39:03.811665+0000","last_active":"2024-09-14T03:40:20.036263+0000","last_peered":"2024-09-14T03:40:20.036263+0000","last_clean":"2024-09-14T03:40:20.036263+0000","last_became_active":"2024-09-14T03:39:01.978728+0000","last_became_peered":"2024-09-14T03:39:01.978728+0000","last_unstale":"2024-09-14T03:40:20.036263+0000","last_undegraded":"2024-09-14T03:40:20.036263+0000","last_fullsized":"2024-09-14T03:40:20.036263+0000","mapping_epoch":15,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":16,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_clean_scrub_stamp":"2024-09-14T03:39:00.791161+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:34:31.446372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00087291699999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":25,"reported_epoch":21,"state":"active+clean","last_fresh":"2024-09-14T03:40:20.062758+0000","last_change":"2024-09-14T03:39:03.819854+0000","last_active":"2024-09-14T03:40:20.062758+0000","last_peered":"2024-09-14T03:40:20.062758+0000","last_clean":"2024-09-14T03:40:20.062758+0000","last_became_active":"2024-09-14T03:39:01.978754+0000","last_became_peered":"2024-09-14T03:39:01.978754+0000","last_unstale":"2024-09-14T03:40:20.062758+0000","last_undegraded":"2024-09-14T03:40:20.062758+0000","last_fullsized":"2024-09-14T03:40:20.062758+0000","mapping_epoch":15,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":16,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_clean_scrub_stamp":"2024-09-14T03:39:00.791161+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:12:43.787489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00048052199999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"18'2","reported_seq":27,"reported_epoch":21,"state":"active+clean","last_fresh":"2024-09-14T03:40:20.036148+0000","last_change":"2024-09-14T03:39:03.812269+0000","last_active":"2024-09-14T03:40:20.036148+0000","last_peered":"2024-09-14T03:40:20.036148+0000","last_clean":"2024-09-14T03:40:20.036148+0000","last_became_active":"2024-09-14T03:39:01.808874+0000","last_became_peered":"2024-09-14T03:39:01.808874+0000","last_unstale":"2024-09-14T03:40:20.036148+0000","last_undegraded":"2024-09-14T03:40:20.036148+0000","last_fullsized":"2024-09-14T03:40:20.036148+0000","mapping_epoch":15,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":16,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_clean_scrub_stamp":"2024-09-14T03:39:00.791161+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:44:11.882619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00135844,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":25,"reported_epoch":21,"state":"active+clean","last_fresh":"2024-09-14T03:40:20.036354+0000","last_change":"2024-09-14T03:39:03.811664+0000","last_active":"2024-09-14T03:40:20.036354+0000","last_peered":"2024-09-14T03:40:20.036354+0000","last_clean":"2024-09-14T03:40:20.036354+0000","last_became_active":"2024-09-14T03:39:01.807757+0000","last_became_peered":"2024-09-14T03:39:01.807757+0000","last_unstale":"2024-09-14T03:40:20.036354+0000","last_undegraded":"2024-09-14T03:40:20.036354+0000","last_fullsized":"2024-09-14T03:40:20.036354+0000","mapping_epoch":15,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":16,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_clean_scrub_stamp":"2024-09-14T03:39:00.791161+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:55:28.670915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046880499999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":25,"reported_epoch":21,"state":"active+clean","last_fresh":"2024-09-14T03:40:20.036049+0000","last_change":"2024-09-14T03:39:03.811316+0000","last_active":"2024-09-14T03:40:20.036049+0000","last_peered":"2024-09-14T03:40:20.036049+0000","last_clean":"2024-09-14T03:40:20.036049+0000","last_became_active":"2024-09-14T03:39:01.810839+0000","last_became_peered":"2024-09-14T03:39:01.810839+0000","last_unstale":"2024-09-14T03:40:20.036049+0000","last_undegraded":"2024-09-14T03:40:20.036049+0000","last_fullsized":"2024-09-14T03:40:20.036049+0000","mapping_epoch":15,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":16,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_clean_scrub_stamp":"2024-09-14T03:39:00.791161+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:53:45.021355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000765802,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"16'1","reported_seq":26,"reported_epoch":21,"state":"active+clean","last_fresh":"2024-09-14T03:40:20.036165+0000","last_change":"2024-09-14T03:39:03.811340+0000","last_active":"2024-09-14T03:40:20.036165+0000","last_peered":"2024-09-14T03:40:20.036165+0000","last_clean":"2024-09-14T03:40:20.036165+0000","last_became_active":"2024-09-14T03:39:01.809115+0000","last_became_peered":"2024-09-14T03:39:01.809115+0000","last_unstale":"2024-09-14T03:40:20.036165+0000","last_undegraded":"2024-09-14T03:40:20.036165+0000","last_fullsized":"2024-09-14T03:40:20.036165+0000","mapping_epoch":15,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":16,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_clean_scrub_stamp":"2024-09-14T03:39:00.791161+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:17:21.729224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00075472199999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"13'97","reported_seq":153,"reported_epoch":21,"state":"active+clean","last_fresh":"2024-09-14T03:40:20.062781+0000","last_change":"2024-09-14T03:38:59.008461+0000","last_active":"2024-09-14T03:40:20.062781+0000","last_peered":"2024-09-14T03:40:20.062781+0000","last_clean":"2024-09-14T03:40:20.062781+0000","last_became_active":"2024-09-14T03:38:59.006003+0000","last_became_peered":"2024-09-14T03:38:59.006003+0000","last_unstale":"2024-09-14T03:40:20.062781+0000","last_undegraded":"2024-09-14T03:40:20.062781+0000","last_fullsized":"2024-09-14T03:40:20.062781+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:38:57.773685+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:57.773685+0000","last_clean_scrub_stamp":"2024-09-14T03:38:57.773685+0000","objects_scrubbed":0,"log_size":97,"log_dups_size":0,"ondisk_log_size":97,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:21:23.620251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":137,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":4},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":137,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":97,"ondisk_log_size":97,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":3,"up_from":11,"seq":47244640275,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27208,"kb_used_data":180,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344632,"statfs":{"total":96636764160,"available":96608903168,"internally_reserved":0,"allocated":184320,"data_stored":52715,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":27654599},"hb_peers":[0,1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":2,"up_from":11,"seq":47244640275,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27204,"kb_used_data":176,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344636,"statfs":{"total":96636764160,"available":96608907264,"internally_reserved":0,"allocated":180224,"data_stored":52696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,1,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":11,"seq":47244640275,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27788,"kb_used_data":760,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344052,"statfs":{"total":96636764160,"available":96608309248,"internally_reserved":0,"allocated":778240,"data_stored":643083,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,2,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":9,"seq":38654705683,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27776,"kb_used_data":748,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344064,"statfs":{"total":96636764160,"available":96608321536,"internally_reserved":0,"allocated":765952,"data_stored":639622,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[1,2,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T03:40:23.529 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-09-14T03:40:23.628 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.626+0000 7fd01734d640 1 -- 172.21.15.83:0/2717600632 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fd010099080 msgr2=0x7fd010099460 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:23.628 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.626+0000 7fd01734d640 1 --2- 172.21.15.83:0/2717600632 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fd010099080 0x7fd010099460 secure :-1 s=READY pgs=34 cs=0 l=1 rev1=1 crypto rx=0x7fd00c009860 tx=0x7fd00c02a460 comp rx=0 tx=0).stop 2024-09-14T03:40:23.629 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.626+0000 7fd01734d640 1 -- 172.21.15.83:0/2717600632 shutdown_connections 2024-09-14T03:40:23.629 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.626+0000 7fd01734d640 1 --2- 172.21.15.83:0/2717600632 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fd01012f1c0 0x7fd01012f610 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:23.629 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.627+0000 7fd01734d640 1 --2- 172.21.15.83:0/2717600632 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fd010099080 0x7fd010099460 unknown :-1 s=CLOSED pgs=34 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:23.629 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.627+0000 7fd01734d640 1 -- 172.21.15.83:0/2717600632 >> 172.21.15.83:0/2717600632 conn(0x7fd0100a1800 msgr2=0x7fd0100a1c10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:23.629 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.627+0000 7fd01734d640 1 -- 172.21.15.83:0/2717600632 shutdown_connections 2024-09-14T03:40:23.629 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.627+0000 7fd01734d640 1 -- 172.21.15.83:0/2717600632 wait complete. 2024-09-14T03:40:23.631 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.627+0000 7fd01734d640 1 Processor -- start 2024-09-14T03:40:23.631 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.627+0000 7fd01734d640 1 -- start start 2024-09-14T03:40:23.631 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.628+0000 7fd01734d640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fd010099080 0x7fd010093450 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:23.631 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.628+0000 7fd01734d640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fd0100999a0 0x7fd010093990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:23.632 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.628+0000 7fd01734d640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fd01012f1c0 0x7fd010093ed0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:23.632 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.628+0000 7fd01734d640 1 -- --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7fd010137660 con 0x7fd010099080 2024-09-14T03:40:23.632 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.628+0000 7fd01734d640 1 -- --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7fd0101374e0 con 0x7fd0100999a0 2024-09-14T03:40:23.632 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.628+0000 7fd01734d640 1 -- --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7fd0101377e0 con 0x7fd01012f1c0 2024-09-14T03:40:23.632 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.628+0000 7fd0150e9640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fd010099080 0x7fd010093450 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:23.632 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.628+0000 7fd0158ea640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fd01012f1c0 0x7fd010093ed0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:23.632 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.628+0000 7fd0148e8640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fd0100999a0 0x7fd010093990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:23.632 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.628+0000 7fd0150e9640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fd010099080 0x7fd010093450 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3300/0 says I am v2:172.21.15.83:40004/0 (socket says 172.21.15.83:40004) 2024-09-14T03:40:23.632 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.628+0000 7fd0158ea640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fd01012f1c0 0x7fd010093ed0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3302/0 says I am v2:172.21.15.83:37846/0 (socket says 172.21.15.83:37846) 2024-09-14T03:40:23.632 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.628+0000 7fd0148e8640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fd0100999a0 0x7fd010093990 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3301/0 says I am v2:172.21.15.83:35742/0 (socket says 172.21.15.83:35742) 2024-09-14T03:40:23.632 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.628+0000 7fd0150e9640 1 -- 172.21.15.83:0/1659001448 learned_addr learned my addr 172.21.15.83:0/1659001448 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:40:23.632 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.628+0000 7fd0158ea640 1 -- 172.21.15.83:0/1659001448 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fd0100999a0 msgr2=0x7fd010093990 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:23.633 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.628+0000 7fd0158ea640 1 --2- 172.21.15.83:0/1659001448 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fd0100999a0 0x7fd010093990 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:23.633 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.628+0000 7fd0158ea640 1 -- 172.21.15.83:0/1659001448 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fd010099080 msgr2=0x7fd010093450 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:23.633 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.628+0000 7fd0158ea640 1 --2- 172.21.15.83:0/1659001448 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fd010099080 0x7fd010093450 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:23.633 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.628+0000 7fd0158ea640 1 -- 172.21.15.83:0/1659001448 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fd0100969e0 con 0x7fd01012f1c0 2024-09-14T03:40:23.633 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.628+0000 7fd0158ea640 1 --2- 172.21.15.83:0/1659001448 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fd01012f1c0 0x7fd010093ed0 secure :-1 s=READY pgs=35 cs=0 l=1 rev1=1 crypto rx=0x7fd00800ec50 tx=0x7fd00800c6a0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:40:23.633 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.628+0000 7fd005ffb640 1 -- 172.21.15.83:0/1659001448 <== mon.2 v2:172.21.15.83:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd008003b70 con 0x7fd01012f1c0 2024-09-14T03:40:23.633 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.628+0000 7fd005ffb640 1 -- 172.21.15.83:0/1659001448 <== mon.2 v2:172.21.15.83:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fd008003d10 con 0x7fd01012f1c0 2024-09-14T03:40:23.633 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.628+0000 7fd01734d640 1 -- 172.21.15.83:0/1659001448 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fd01009ebe0 con 0x7fd01012f1c0 2024-09-14T03:40:23.633 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.629+0000 7fd005ffb640 1 -- 172.21.15.83:0/1659001448 <== mon.2 v2:172.21.15.83:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd008004280 con 0x7fd01012f1c0 2024-09-14T03:40:23.633 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.629+0000 7fd01734d640 1 -- 172.21.15.83:0/1659001448 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7fd010094ad0 con 0x7fd01012f1c0 2024-09-14T03:40:23.633 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.630+0000 7fd01734d640 1 -- 172.21.15.83:0/1659001448 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fd010099460 con 0x7fd01012f1c0 2024-09-14T03:40:23.633 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.630+0000 7fd005ffb640 1 -- 172.21.15.83:0/1659001448 <== mon.2 v2:172.21.15.83:3302/0 4 ==== mgrmap(e 9) ==== 63442+0+0 (secure 0 0 0) 0x7fd008004420 con 0x7fd01012f1c0 2024-09-14T03:40:23.633 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.630+0000 7fd005ffb640 1 -- 172.21.15.83:0/1659001448 <== mon.2 v2:172.21.15.83:3302/0 5 ==== osd_map(21..21 src has 1..21) ==== 3027+0+0 (secure 0 0 0) 0x7fd008069c30 con 0x7fd01012f1c0 2024-09-14T03:40:23.635 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.633+0000 7fd005ffb640 1 -- 172.21.15.83:0/1659001448 <== mon.2 v2:172.21.15.83:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+157506 (secure 0 0 0) 0x7fd008004710 con 0x7fd01012f1c0 2024-09-14T03:40:23.780 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.778+0000 7fd01734d640 1 -- 172.21.15.83:0/1659001448 --> v1:172.21.15.83:6800/1803897559 -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) -- 0x7fd0100698c0 con 0x7fcff4051450 2024-09-14T03:40:23.780 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.779+0000 7fd005ffb640 1 -- 172.21.15.83:0/1659001448 <== mgr.14106 v1:172.21.15.83:6800/1803897559 1 ==== mgr_command_reply(tid 0: 0 dumped all) ==== 18+0+32011 (unknown 2965378022 0 2510186030) 0x7fd0100698c0 con 0x7fcff4051450 2024-09-14T03:40:23.781 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:40:23.781 INFO:teuthology.orchestra.run.smithi083.stderr:dumped all 2024-09-14T03:40:23.782 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.781+0000 7fd01734d640 1 -- 172.21.15.83:0/1659001448 >> v1:172.21.15.83:6800/1803897559 conn(0x7fcff4051450 legacy=0x7fcff4053910 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:23.782 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.781+0000 7fd01734d640 1 -- 172.21.15.83:0/1659001448 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fd01012f1c0 msgr2=0x7fd010093ed0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:23.782 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.781+0000 7fd01734d640 1 --2- 172.21.15.83:0/1659001448 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fd01012f1c0 0x7fd010093ed0 secure :-1 s=READY pgs=35 cs=0 l=1 rev1=1 crypto rx=0x7fd00800ec50 tx=0x7fd00800c6a0 comp rx=0 tx=0).stop 2024-09-14T03:40:23.783 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.781+0000 7fd01734d640 1 -- 172.21.15.83:0/1659001448 shutdown_connections 2024-09-14T03:40:23.783 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.781+0000 7fd01734d640 1 --2- 172.21.15.83:0/1659001448 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fd01012f1c0 0x7fd010093ed0 unknown :-1 s=CLOSED pgs=35 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:23.783 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.781+0000 7fd01734d640 1 --2- 172.21.15.83:0/1659001448 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fd0100999a0 0x7fd010093990 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:23.783 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.781+0000 7fd01734d640 1 --2- 172.21.15.83:0/1659001448 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fd010099080 0x7fd010093450 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:23.783 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.781+0000 7fd01734d640 1 -- 172.21.15.83:0/1659001448 >> 172.21.15.83:0/1659001448 conn(0x7fd0100a1800 msgr2=0x7fd010097a40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:23.783 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.781+0000 7fd01734d640 1 -- 172.21.15.83:0/1659001448 shutdown_connections 2024-09-14T03:40:23.783 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.781+0000 7fd01734d640 1 -- 172.21.15.83:0/1659001448 wait complete. 2024-09-14T03:40:23.793 INFO:teuthology.orchestra.run.smithi083.stdout:{"pg_ready":true,"pg_map":{"version":4,"stamp":"2024-09-14T03:40:22.039693+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":139,"num_write_kb":1896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":100,"ondisk_log_size":100,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":4,"num_per_pool_osds":4,"num_per_pool_omap_osds":4,"kb":377487360,"kb_used":109976,"kb_used_data":1864,"kb_used_omap":6,"kb_used_meta":108025,"kb_avail":377377384,"statfs":{"total":386547056640,"available":386434441216,"internally_reserved":0,"allocated":1908736,"data_stored":1388116,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6378,"internal_metadata":110618390},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"0.997195"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":25,"reported_epoch":21,"state":"active+clean","last_fresh":"2024-09-14T03:40:20.036336+0000","last_change":"2024-09-14T03:39:03.811650+0000","last_active":"2024-09-14T03:40:20.036336+0000","last_peered":"2024-09-14T03:40:20.036336+0000","last_clean":"2024-09-14T03:40:20.036336+0000","last_became_active":"2024-09-14T03:39:01.978698+0000","last_became_peered":"2024-09-14T03:39:01.978698+0000","last_unstale":"2024-09-14T03:40:20.036336+0000","last_undegraded":"2024-09-14T03:40:20.036336+0000","last_fullsized":"2024-09-14T03:40:20.036336+0000","mapping_epoch":15,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":16,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_clean_scrub_stamp":"2024-09-14T03:39:00.791161+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:14:21.446815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00098140899999999993,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":25,"reported_epoch":21,"state":"active+clean","last_fresh":"2024-09-14T03:40:20.062773+0000","last_change":"2024-09-14T03:39:03.819819+0000","last_active":"2024-09-14T03:40:20.062773+0000","last_peered":"2024-09-14T03:40:20.062773+0000","last_clean":"2024-09-14T03:40:20.062773+0000","last_became_active":"2024-09-14T03:39:01.978827+0000","last_became_peered":"2024-09-14T03:39:01.978827+0000","last_unstale":"2024-09-14T03:40:20.062773+0000","last_undegraded":"2024-09-14T03:40:20.062773+0000","last_fullsized":"2024-09-14T03:40:20.062773+0000","mapping_epoch":15,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":16,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_clean_scrub_stamp":"2024-09-14T03:39:00.791161+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:40:25.557607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00045411099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":25,"reported_epoch":21,"state":"active+clean","last_fresh":"2024-09-14T03:40:20.036263+0000","last_change":"2024-09-14T03:39:03.811665+0000","last_active":"2024-09-14T03:40:20.036263+0000","last_peered":"2024-09-14T03:40:20.036263+0000","last_clean":"2024-09-14T03:40:20.036263+0000","last_became_active":"2024-09-14T03:39:01.978728+0000","last_became_peered":"2024-09-14T03:39:01.978728+0000","last_unstale":"2024-09-14T03:40:20.036263+0000","last_undegraded":"2024-09-14T03:40:20.036263+0000","last_fullsized":"2024-09-14T03:40:20.036263+0000","mapping_epoch":15,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":16,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_clean_scrub_stamp":"2024-09-14T03:39:00.791161+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:34:31.446372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00087291699999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":25,"reported_epoch":21,"state":"active+clean","last_fresh":"2024-09-14T03:40:20.062758+0000","last_change":"2024-09-14T03:39:03.819854+0000","last_active":"2024-09-14T03:40:20.062758+0000","last_peered":"2024-09-14T03:40:20.062758+0000","last_clean":"2024-09-14T03:40:20.062758+0000","last_became_active":"2024-09-14T03:39:01.978754+0000","last_became_peered":"2024-09-14T03:39:01.978754+0000","last_unstale":"2024-09-14T03:40:20.062758+0000","last_undegraded":"2024-09-14T03:40:20.062758+0000","last_fullsized":"2024-09-14T03:40:20.062758+0000","mapping_epoch":15,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":16,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_clean_scrub_stamp":"2024-09-14T03:39:00.791161+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:12:43.787489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00048052199999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"18'2","reported_seq":27,"reported_epoch":21,"state":"active+clean","last_fresh":"2024-09-14T03:40:20.036148+0000","last_change":"2024-09-14T03:39:03.812269+0000","last_active":"2024-09-14T03:40:20.036148+0000","last_peered":"2024-09-14T03:40:20.036148+0000","last_clean":"2024-09-14T03:40:20.036148+0000","last_became_active":"2024-09-14T03:39:01.808874+0000","last_became_peered":"2024-09-14T03:39:01.808874+0000","last_unstale":"2024-09-14T03:40:20.036148+0000","last_undegraded":"2024-09-14T03:40:20.036148+0000","last_fullsized":"2024-09-14T03:40:20.036148+0000","mapping_epoch":15,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":16,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_clean_scrub_stamp":"2024-09-14T03:39:00.791161+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:44:11.882619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00135844,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":25,"reported_epoch":21,"state":"active+clean","last_fresh":"2024-09-14T03:40:20.036354+0000","last_change":"2024-09-14T03:39:03.811664+0000","last_active":"2024-09-14T03:40:20.036354+0000","last_peered":"2024-09-14T03:40:20.036354+0000","last_clean":"2024-09-14T03:40:20.036354+0000","last_became_active":"2024-09-14T03:39:01.807757+0000","last_became_peered":"2024-09-14T03:39:01.807757+0000","last_unstale":"2024-09-14T03:40:20.036354+0000","last_undegraded":"2024-09-14T03:40:20.036354+0000","last_fullsized":"2024-09-14T03:40:20.036354+0000","mapping_epoch":15,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":16,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_clean_scrub_stamp":"2024-09-14T03:39:00.791161+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:55:28.670915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046880499999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":25,"reported_epoch":21,"state":"active+clean","last_fresh":"2024-09-14T03:40:20.036049+0000","last_change":"2024-09-14T03:39:03.811316+0000","last_active":"2024-09-14T03:40:20.036049+0000","last_peered":"2024-09-14T03:40:20.036049+0000","last_clean":"2024-09-14T03:40:20.036049+0000","last_became_active":"2024-09-14T03:39:01.810839+0000","last_became_peered":"2024-09-14T03:39:01.810839+0000","last_unstale":"2024-09-14T03:40:20.036049+0000","last_undegraded":"2024-09-14T03:40:20.036049+0000","last_fullsized":"2024-09-14T03:40:20.036049+0000","mapping_epoch":15,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":16,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_clean_scrub_stamp":"2024-09-14T03:39:00.791161+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:53:45.021355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000765802,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"16'1","reported_seq":26,"reported_epoch":21,"state":"active+clean","last_fresh":"2024-09-14T03:40:20.036165+0000","last_change":"2024-09-14T03:39:03.811340+0000","last_active":"2024-09-14T03:40:20.036165+0000","last_peered":"2024-09-14T03:40:20.036165+0000","last_clean":"2024-09-14T03:40:20.036165+0000","last_became_active":"2024-09-14T03:39:01.809115+0000","last_became_peered":"2024-09-14T03:39:01.809115+0000","last_unstale":"2024-09-14T03:40:20.036165+0000","last_undegraded":"2024-09-14T03:40:20.036165+0000","last_fullsized":"2024-09-14T03:40:20.036165+0000","mapping_epoch":15,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":16,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_clean_scrub_stamp":"2024-09-14T03:39:00.791161+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:17:21.729224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00075472199999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"13'97","reported_seq":153,"reported_epoch":21,"state":"active+clean","last_fresh":"2024-09-14T03:40:20.062781+0000","last_change":"2024-09-14T03:38:59.008461+0000","last_active":"2024-09-14T03:40:20.062781+0000","last_peered":"2024-09-14T03:40:20.062781+0000","last_clean":"2024-09-14T03:40:20.062781+0000","last_became_active":"2024-09-14T03:38:59.006003+0000","last_became_peered":"2024-09-14T03:38:59.006003+0000","last_unstale":"2024-09-14T03:40:20.062781+0000","last_undegraded":"2024-09-14T03:40:20.062781+0000","last_fullsized":"2024-09-14T03:40:20.062781+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:38:57.773685+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:57.773685+0000","last_clean_scrub_stamp":"2024-09-14T03:38:57.773685+0000","objects_scrubbed":0,"log_size":97,"log_dups_size":0,"ondisk_log_size":97,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:21:23.620251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":137,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":4},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":137,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":97,"ondisk_log_size":97,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":3,"up_from":11,"seq":47244640275,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27208,"kb_used_data":180,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344632,"statfs":{"total":96636764160,"available":96608903168,"internally_reserved":0,"allocated":184320,"data_stored":52715,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":27654599},"hb_peers":[0,1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":2,"up_from":11,"seq":47244640275,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27204,"kb_used_data":176,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344636,"statfs":{"total":96636764160,"available":96608907264,"internally_reserved":0,"allocated":180224,"data_stored":52696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,1,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":11,"seq":47244640275,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27788,"kb_used_data":760,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344052,"statfs":{"total":96636764160,"available":96608309248,"internally_reserved":0,"allocated":778240,"data_stored":643083,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,2,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":9,"seq":38654705683,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27776,"kb_used_data":748,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344064,"statfs":{"total":96636764160,"available":96608321536,"internally_reserved":0,"allocated":765952,"data_stored":639622,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[1,2,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T03:40:23.794 INFO:tasks.ceph.ceph_manager.ceph:clean! 2024-09-14T03:40:23.794 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-09-14T03:40:23.794 INFO:tasks.ceph.ceph_manager.ceph:wait_until_healthy 2024-09-14T03:40:23.794 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph health --format=json 2024-09-14T03:40:23.893 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.891+0000 7fdba4a52640 1 -- 172.21.15.83:0/744535091 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fdba0095c20 msgr2=0x7fdba0096080 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:23.893 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.891+0000 7fdba4a52640 1 --2- 172.21.15.83:0/744535091 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fdba0095c20 0x7fdba0096080 secure :-1 s=READY pgs=36 cs=0 l=1 rev1=1 crypto rx=0x7fdb98005fd0 tx=0x7fdb9802a6e0 comp rx=0 tx=0).stop 2024-09-14T03:40:23.894 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.891+0000 7fdba4a52640 1 -- 172.21.15.83:0/744535091 shutdown_connections 2024-09-14T03:40:23.894 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.891+0000 7fdba4a52640 1 --2- 172.21.15.83:0/744535091 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fdba0095c20 0x7fdba0096080 unknown :-1 s=CLOSED pgs=36 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:23.894 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.891+0000 7fdba4a52640 1 --2- 172.21.15.83:0/744535091 >> v2:172.21.15.83:3300/0 conn(0x7fdba012c160 0x7fdba00956e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:23.894 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.891+0000 7fdba4a52640 1 --2- 172.21.15.83:0/744535091 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fdba00991f0 0x7fdba012bc20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:23.894 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.891+0000 7fdba4a52640 1 -- 172.21.15.83:0/744535091 >> 172.21.15.83:0/744535091 conn(0x7fdba009a2e0 msgr2=0x7fdba0097ad0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:23.894 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.891+0000 7fdba4a52640 1 -- 172.21.15.83:0/744535091 shutdown_connections 2024-09-14T03:40:23.895 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.891+0000 7fdba4a52640 1 -- 172.21.15.83:0/744535091 wait complete. 2024-09-14T03:40:23.897 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.892+0000 7fdba4a52640 1 Processor -- start 2024-09-14T03:40:23.897 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.892+0000 7fdba4a52640 1 -- start start 2024-09-14T03:40:23.897 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.893+0000 7fdba4a52640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fdba0095c20 0x7fdba01306e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:23.897 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.893+0000 7fdba4a52640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fdba00991f0 0x7fdba0130c20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:23.897 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.893+0000 7fdba4a52640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fdba012c160 0x7fdba0131160 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:23.897 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.893+0000 7fdba4a52640 1 -- --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7fdba0133530 con 0x7fdba0095c20 2024-09-14T03:40:23.897 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.893+0000 7fdba4a52640 1 -- --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7fdba01333b0 con 0x7fdba012c160 2024-09-14T03:40:23.897 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.893+0000 7fdba4a52640 1 -- --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7fdba01336b0 con 0x7fdba00991f0 2024-09-14T03:40:23.898 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.893+0000 7fdb9ed9d640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fdba012c160 0x7fdba0131160 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:23.898 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.893+0000 7fdb9e59c640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fdba0095c20 0x7fdba01306e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:23.898 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.893+0000 7fdb97fff640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fdba00991f0 0x7fdba0130c20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:23.898 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.893+0000 7fdb9ed9d640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fdba012c160 0x7fdba0131160 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3301/0 says I am v2:172.21.15.83:35760/0 (socket says 172.21.15.83:35760) 2024-09-14T03:40:23.898 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.893+0000 7fdb9ed9d640 1 -- 172.21.15.83:0/1165088849 learned_addr learned my addr 172.21.15.83:0/1165088849 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:40:23.898 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.893+0000 7fdb9e59c640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fdba0095c20 0x7fdba01306e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3300/0 says I am v2:172.21.15.83:40020/0 (socket says 172.21.15.83:40020) 2024-09-14T03:40:23.898 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.893+0000 7fdb97fff640 1 -- 172.21.15.83:0/1165088849 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fdba012c160 msgr2=0x7fdba0131160 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:23.898 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.893+0000 7fdb97fff640 1 --2- 172.21.15.83:0/1165088849 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fdba012c160 0x7fdba0131160 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:23.898 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.893+0000 7fdb97fff640 1 -- 172.21.15.83:0/1165088849 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fdba0095c20 msgr2=0x7fdba01306e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:23.898 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.893+0000 7fdb97fff640 1 --2- 172.21.15.83:0/1165088849 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fdba0095c20 0x7fdba01306e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:23.898 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.893+0000 7fdb97fff640 1 -- 172.21.15.83:0/1165088849 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fdba0094890 con 0x7fdba00991f0 2024-09-14T03:40:23.898 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.893+0000 7fdb97fff640 1 --2- 172.21.15.83:0/1165088849 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fdba00991f0 0x7fdba0130c20 secure :-1 s=READY pgs=37 cs=0 l=1 rev1=1 crypto rx=0x7fdb88009ce0 tx=0x7fdb8800b690 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:40:23.898 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.893+0000 7fdb96ffd640 1 -- 172.21.15.83:0/1165088849 <== mon.2 v2:172.21.15.83:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fdb88004020 con 0x7fdba00991f0 2024-09-14T03:40:23.899 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.893+0000 7fdb96ffd640 1 -- 172.21.15.83:0/1165088849 <== mon.2 v2:172.21.15.83:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fdb880041c0 con 0x7fdba00991f0 2024-09-14T03:40:23.899 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.894+0000 7fdba4a52640 1 -- 172.21.15.83:0/1165088849 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fdba0094f00 con 0x7fdba00991f0 2024-09-14T03:40:23.899 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.894+0000 7fdb96ffd640 1 -- 172.21.15.83:0/1165088849 <== mon.2 v2:172.21.15.83:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fdb880035f0 con 0x7fdba00991f0 2024-09-14T03:40:23.899 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.894+0000 7fdba4a52640 1 -- 172.21.15.83:0/1165088849 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7fdba01c29a0 con 0x7fdba00991f0 2024-09-14T03:40:23.899 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.895+0000 7fdba4a52640 1 -- 172.21.15.83:0/1165088849 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fdba012bc20 con 0x7fdba00991f0 2024-09-14T03:40:23.899 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.895+0000 7fdb96ffd640 1 -- 172.21.15.83:0/1165088849 <== mon.2 v2:172.21.15.83:3302/0 4 ==== mgrmap(e 9) ==== 63442+0+0 (secure 0 0 0) 0x7fdb88009310 con 0x7fdba00991f0 2024-09-14T03:40:23.899 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.895+0000 7fdb96ffd640 1 -- 172.21.15.83:0/1165088849 <== mon.2 v2:172.21.15.83:3302/0 5 ==== osd_map(21..21 src has 1..21) ==== 3027+0+0 (secure 0 0 0) 0x7fdb8806ce70 con 0x7fdba00991f0 2024-09-14T03:40:23.900 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:23.898+0000 7fdb96ffd640 1 -- 172.21.15.83:0/1165088849 <== mon.2 v2:172.21.15.83:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+157506 (secure 0 0 0) 0x7fdb8806b750 con 0x7fdba00991f0 2024-09-14T03:40:24.055 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:24.054+0000 7fdb96ffd640 1 -- 172.21.15.83:0/1165088849 <== mon.2 v2:172.21.15.83:3302/0 7 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7fdb8806bad0 con 0x7fdba00991f0 2024-09-14T03:40:24.080 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:24.078+0000 7fdba4a52640 1 -- 172.21.15.83:0/1165088849 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_command({"prefix": "health", "format": "json"} v 0) -- 0x7fdba0131e10 con 0x7fdba00991f0 2024-09-14T03:40:24.080 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:24.078+0000 7fdb96ffd640 1 -- 172.21.15.83:0/1165088849 <== mon.2 v2:172.21.15.83:3302/0 8 ==== mon_command_ack([{"prefix": "health", "format": "json"}]=0 v0) ==== 72+0+46 (secure 0 0 0) 0x7fdb8804d910 con 0x7fdba00991f0 2024-09-14T03:40:24.080 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:40:24.080 INFO:teuthology.orchestra.run.smithi083.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-09-14T03:40:24.082 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:24.080+0000 7fdba4a52640 1 -- 172.21.15.83:0/1165088849 >> v1:172.21.15.83:6800/1803897559 conn(0x7fdb80051420 legacy=0x7fdb800538e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:24.082 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:24.080+0000 7fdba4a52640 1 -- 172.21.15.83:0/1165088849 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fdba00991f0 msgr2=0x7fdba0130c20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:24.083 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:24.080+0000 7fdba4a52640 1 --2- 172.21.15.83:0/1165088849 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fdba00991f0 0x7fdba0130c20 secure :-1 s=READY pgs=37 cs=0 l=1 rev1=1 crypto rx=0x7fdb88009ce0 tx=0x7fdb8800b690 comp rx=0 tx=0).stop 2024-09-14T03:40:24.083 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:24.081+0000 7fdba4a52640 1 -- 172.21.15.83:0/1165088849 shutdown_connections 2024-09-14T03:40:24.083 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:24.081+0000 7fdba4a52640 1 --2- 172.21.15.83:0/1165088849 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fdba012c160 0x7fdba0131160 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:24.083 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:24.081+0000 7fdba4a52640 1 --2- 172.21.15.83:0/1165088849 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fdba00991f0 0x7fdba0130c20 unknown :-1 s=CLOSED pgs=37 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:24.083 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:24.081+0000 7fdba4a52640 1 --2- 172.21.15.83:0/1165088849 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fdba0095c20 0x7fdba01306e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:24.083 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:24.081+0000 7fdba4a52640 1 -- 172.21.15.83:0/1165088849 >> 172.21.15.83:0/1165088849 conn(0x7fdba009a2e0 msgr2=0x7fdba0092af0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:24.083 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:24.081+0000 7fdba4a52640 1 -- 172.21.15.83:0/1165088849 shutdown_connections 2024-09-14T03:40:24.083 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:24.081+0000 7fdba4a52640 1 -- 172.21.15.83:0/1165088849 wait complete. 2024-09-14T03:40:24.093 INFO:tasks.ceph.ceph_manager.ceph:wait_until_healthy done 2024-09-14T03:40:24.093 INFO:teuthology.run_tasks:Running task ceph.restart... 2024-09-14T03:40:24.106 DEBUG:tasks.ceph.osd.0:waiting for process to exit 2024-09-14T03:40:24.106 INFO:teuthology.orchestra.run:waiting for 300 2024-09-14T03:40:24.141 INFO:tasks.ceph.osd.0:Stopped 2024-09-14T03:40:24.141 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd down 0 2024-09-14T03:40:24.282 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:24.281+0000 7f579ce0f640 1 -- 172.21.15.83:0/1491937539 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f5798099660 msgr2=0x7f579812eab0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:24.283 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:24.281+0000 7f579ce0f640 1 --2- 172.21.15.83:0/1491937539 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f5798099660 0x7f579812eab0 secure :-1 s=READY pgs=38 cs=0 l=1 rev1=1 crypto rx=0x7f578c009830 tx=0x7f578c02a360 comp rx=0 tx=0).stop 2024-09-14T03:40:24.283 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:24.281+0000 7f579ce0f640 1 -- 172.21.15.83:0/1491937539 shutdown_connections 2024-09-14T03:40:24.283 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:24.281+0000 7f579ce0f640 1 --2- 172.21.15.83:0/1491937539 >> v2:172.21.15.83:3300/0 conn(0x7f579812eff0 0x7f579812f480 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:24.283 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:24.281+0000 7f579ce0f640 1 --2- 172.21.15.83:0/1491937539 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f5798099660 0x7f579812eab0 unknown :-1 s=CLOSED pgs=38 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:24.283 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:24.281+0000 7f579ce0f640 1 --2- 172.21.15.83:0/1491937539 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f5798098d40 0x7f5798099120 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:24.283 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:24.281+0000 7f579ce0f640 1 -- 172.21.15.83:0/1491937539 >> 172.21.15.83:0/1491937539 conn(0x7f57980a1790 msgr2=0x7f57980a1ba0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:24.283 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:24.281+0000 7f579ce0f640 1 -- 172.21.15.83:0/1491937539 shutdown_connections 2024-09-14T03:40:24.283 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:24.281+0000 7f579ce0f640 1 -- 172.21.15.83:0/1491937539 wait complete. 2024-09-14T03:40:24.285 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:24.282+0000 7f579ce0f640 1 Processor -- start 2024-09-14T03:40:24.285 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:24.282+0000 7f579ce0f640 1 -- start start 2024-09-14T03:40:24.285 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:24.282+0000 7f579ce0f640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f5798098d40 0x7f57981c4ce0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:24.285 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:24.282+0000 7f579ce0f640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f5798099660 0x7f57981c5220 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:24.285 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:24.282+0000 7f579ce0f640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f579812eff0 0x7f5798132be0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:24.286 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:24.282+0000 7f579ce0f640 1 -- --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7f5798136610 con 0x7f5798099660 2024-09-14T03:40:24.286 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:24.282+0000 7f579ce0f640 1 -- --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7f5798136490 con 0x7f5798098d40 2024-09-14T03:40:24.286 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:24.282+0000 7f579ce0f640 1 -- --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7f5798136790 con 0x7f579812eff0 2024-09-14T03:40:24.286 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:24.282+0000 7f579659c640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f5798098d40 0x7f57981c4ce0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:24.286 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:24.282+0000 7f579659c640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f5798098d40 0x7f57981c4ce0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3301/0 says I am v2:172.21.15.83:35772/0 (socket says 172.21.15.83:35772) 2024-09-14T03:40:24.286 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:24.282+0000 7f579659c640 1 -- 172.21.15.83:0/2828940391 learned_addr learned my addr 172.21.15.83:0/2828940391 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:40:24.286 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:24.282+0000 7f5795d9b640 1 --2- 172.21.15.83:0/2828940391 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f5798099660 0x7f57981c5220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:24.286 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:24.282+0000 7f5796d9d640 1 --2- 172.21.15.83:0/2828940391 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f579812eff0 0x7f5798132be0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:24.286 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:24.282+0000 7f5795d9b640 1 -- 172.21.15.83:0/2828940391 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f579812eff0 msgr2=0x7f5798132be0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:24.286 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:24.282+0000 7f5795d9b640 1 --2- 172.21.15.83:0/2828940391 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f579812eff0 0x7f5798132be0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:24.286 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:24.282+0000 7f5795d9b640 1 -- 172.21.15.83:0/2828940391 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f5798098d40 msgr2=0x7f57981c4ce0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:24.286 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:24.282+0000 7f5795d9b640 1 --2- 172.21.15.83:0/2828940391 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f5798098d40 0x7f57981c4ce0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:24.286 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:24.282+0000 7f5795d9b640 1 -- 172.21.15.83:0/2828940391 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f5798073ee0 con 0x7f5798099660 2024-09-14T03:40:24.286 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:24.283+0000 7f5796d9d640 1 --2- 172.21.15.83:0/2828940391 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f579812eff0 0x7f5798132be0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:40:24.287 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:24.283+0000 7f579659c640 1 --2- 172.21.15.83:0/2828940391 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f5798098d40 0x7f57981c4ce0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:40:24.287 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:24.283+0000 7f5795d9b640 1 --2- 172.21.15.83:0/2828940391 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f5798099660 0x7f57981c5220 secure :-1 s=READY pgs=50 cs=0 l=1 rev1=1 crypto rx=0x7f578c031000 tx=0x7f578c033b70 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:40:24.287 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:24.283+0000 7f5786ffd640 1 -- 172.21.15.83:0/2828940391 <== mon.0 v2:172.21.15.83:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f578c03f020 con 0x7f5798099660 2024-09-14T03:40:24.287 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:24.283+0000 7f5786ffd640 1 -- 172.21.15.83:0/2828940391 <== mon.0 v2:172.21.15.83:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f578c033d60 con 0x7f5798099660 2024-09-14T03:40:24.287 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:24.283+0000 7f579ce0f640 1 -- 172.21.15.83:0/2828940391 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f5798134cc0 con 0x7f5798099660 2024-09-14T03:40:24.287 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:24.283+0000 7f5786ffd640 1 -- 172.21.15.83:0/2828940391 <== mon.0 v2:172.21.15.83:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f578c037040 con 0x7f5798099660 2024-09-14T03:40:24.287 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:24.283+0000 7f579ce0f640 1 -- 172.21.15.83:0/2828940391 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f57981c5760 con 0x7f5798099660 2024-09-14T03:40:24.287 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:24.284+0000 7f5786ffd640 1 -- 172.21.15.83:0/2828940391 <== mon.0 v2:172.21.15.83:3300/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7f578c042020 con 0x7f5798099660 2024-09-14T03:40:24.287 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:24.284+0000 7f579ce0f640 1 -- 172.21.15.83:0/2828940391 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f579812e240 con 0x7f5798099660 2024-09-14T03:40:24.287 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:24.284+0000 7f5786ffd640 1 -- 172.21.15.83:0/2828940391 <== mon.0 v2:172.21.15.83:3300/0 5 ==== osd_map(21..21 src has 1..21) ==== 3027+0+0 (secure 0 0 0) 0x7f578c088d40 con 0x7f5798099660 2024-09-14T03:40:24.289 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:24.288+0000 7f5786ffd640 1 -- 172.21.15.83:0/2828940391 <== mon.0 v2:172.21.15.83:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+157506 (secure 0 0 0) 0x7f579812e240 con 0x7f5798099660 2024-09-14T03:40:24.445 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:24.443+0000 7f579ce0f640 1 -- 172.21.15.83:0/2828940391 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_command({"prefix": "osd down", "ids": ["0"]} v 0) -- 0x7f57980698c0 con 0x7f5798099660 2024-09-14T03:40:24.445 INFO:tasks.ceph.mon.a.smithi083.stderr:2024-09-14T03:40:24.443+0000 7f14662ef640 -1 mon.a@0(leader).osd e21 definitely_dead 0 2024-09-14T03:40:25.056 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:25.054+0000 7f5786ffd640 1 -- 172.21.15.83:0/2828940391 <== mon.0 v2:172.21.15.83:3300/0 7 ==== mon_command_ack([{"prefix": "osd down", "ids": ["0"]}]=0 marked down osd.0. v22) ==== 89+0+0 (secure 0 0 0) 0x7f578c049030 con 0x7f5798099660 2024-09-14T03:40:25.056 INFO:teuthology.orchestra.run.smithi083.stderr:marked down osd.0. 2024-09-14T03:40:25.059 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:25.057+0000 7f579ce0f640 1 -- 172.21.15.83:0/2828940391 >> v1:172.21.15.83:6800/1803897559 conn(0x7f57780514b0 legacy=0x7f5778053970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:25.059 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:25.057+0000 7f579ce0f640 1 -- 172.21.15.83:0/2828940391 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f5798099660 msgr2=0x7f57981c5220 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:25.059 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:25.057+0000 7f579ce0f640 1 --2- 172.21.15.83:0/2828940391 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f5798099660 0x7f57981c5220 secure :-1 s=READY pgs=50 cs=0 l=1 rev1=1 crypto rx=0x7f578c031000 tx=0x7f578c033b70 comp rx=0 tx=0).stop 2024-09-14T03:40:25.059 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:25.057+0000 7f579ce0f640 1 -- 172.21.15.83:0/2828940391 shutdown_connections 2024-09-14T03:40:25.059 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:25.057+0000 7f579ce0f640 1 --2- 172.21.15.83:0/2828940391 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f579812eff0 0x7f5798132be0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:25.059 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:25.057+0000 7f579ce0f640 1 --2- 172.21.15.83:0/2828940391 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f5798099660 0x7f57981c5220 unknown :-1 s=CLOSED pgs=50 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:25.059 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:25.057+0000 7f579ce0f640 1 --2- 172.21.15.83:0/2828940391 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f5798098d40 0x7f57981c4ce0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:25.059 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:25.057+0000 7f579ce0f640 1 -- 172.21.15.83:0/2828940391 >> 172.21.15.83:0/2828940391 conn(0x7f57980a1790 msgr2=0x7f5798128440 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:25.060 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:25.058+0000 7f579ce0f640 1 -- 172.21.15.83:0/2828940391 shutdown_connections 2024-09-14T03:40:25.060 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:25.058+0000 7f579ce0f640 1 -- 172.21.15.83:0/2828940391 wait complete. 2024-09-14T03:40:25.072 INFO:tasks.ceph.osd.0:Restarting daemon 2024-09-14T03:40:25.072 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster ceph -i 0 2024-09-14T03:40:25.075 INFO:tasks.ceph.osd.0:Started 2024-09-14T03:40:25.075 DEBUG:tasks.ceph.osd.1:waiting for process to exit 2024-09-14T03:40:25.075 INFO:teuthology.orchestra.run:waiting for 300 2024-09-14T03:40:25.109 INFO:tasks.ceph.osd.1:Stopped 2024-09-14T03:40:25.109 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd down 1 2024-09-14T03:40:25.251 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:25.250+0000 7fec5dcd4640 1 -- 172.21.15.83:0/3023079334 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fec58098d40 msgr2=0x7fec58099120 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:25.252 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:25.250+0000 7fec5dcd4640 1 --2- 172.21.15.83:0/3023079334 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fec58098d40 0x7fec58099120 secure :-1 s=READY pgs=35 cs=0 l=1 rev1=1 crypto rx=0x7fec48009be0 tx=0x7fec4802a770 comp rx=0 tx=0).stop 2024-09-14T03:40:25.252 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:25.250+0000 7fec5dcd4640 1 -- 172.21.15.83:0/3023079334 shutdown_connections 2024-09-14T03:40:25.252 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:25.250+0000 7fec5dcd4640 1 --2- 172.21.15.83:0/3023079334 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fec5812f1c0 0x7fec5812f650 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:25.252 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:25.250+0000 7fec5dcd4640 1 --2- 172.21.15.83:0/3023079334 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fec58098d40 0x7fec58099120 unknown :-1 s=CLOSED pgs=35 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:25.252 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:25.250+0000 7fec5dcd4640 1 -- 172.21.15.83:0/3023079334 >> 172.21.15.83:0/3023079334 conn(0x7fec580a1790 msgr2=0x7fec580a1ba0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:25.252 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:25.250+0000 7fec5dcd4640 1 -- 172.21.15.83:0/3023079334 shutdown_connections 2024-09-14T03:40:25.252 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:25.250+0000 7fec5dcd4640 1 -- 172.21.15.83:0/3023079334 wait complete. 2024-09-14T03:40:25.254 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:25.251+0000 7fec5dcd4640 1 Processor -- start 2024-09-14T03:40:25.255 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:25.251+0000 7fec5dcd4640 1 -- start start 2024-09-14T03:40:25.255 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:25.251+0000 7fec5dcd4640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fec58098d40 0x7fec581c4ef0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:25.255 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:25.251+0000 7fec5dcd4640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fec58099660 0x7fec581c5430 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:25.255 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:25.251+0000 7fec5dcd4640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fec5812f1c0 0x7fec58071520 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:25.255 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:25.251+0000 7fec5dcd4640 1 -- --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7fec581378c0 con 0x7fec58099660 2024-09-14T03:40:25.255 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:25.251+0000 7fec5dcd4640 1 -- --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7fec58137740 con 0x7fec58098d40 2024-09-14T03:40:25.255 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:25.251+0000 7fec5dcd4640 1 -- --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7fec58137a40 con 0x7fec5812f1c0 2024-09-14T03:40:25.255 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:25.251+0000 7fec577fe640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fec58098d40 0x7fec581c4ef0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:25.255 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:25.251+0000 7fec56ffd640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fec58099660 0x7fec581c5430 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:25.255 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:25.251+0000 7fec577fe640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fec58098d40 0x7fec581c4ef0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3301/0 says I am v2:172.21.15.83:35804/0 (socket says 172.21.15.83:35804) 2024-09-14T03:40:25.255 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:25.251+0000 7fec577fe640 1 -- 172.21.15.83:0/3981728111 learned_addr learned my addr 172.21.15.83:0/3981728111 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:40:25.255 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:25.251+0000 7fec57fff640 1 --2- 172.21.15.83:0/3981728111 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fec5812f1c0 0x7fec58071520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:25.255 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:25.252+0000 7fec577fe640 1 -- 172.21.15.83:0/3981728111 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fec5812f1c0 msgr2=0x7fec58071520 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:25.256 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:25.252+0000 7fec577fe640 1 --2- 172.21.15.83:0/3981728111 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fec5812f1c0 0x7fec58071520 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:25.256 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:25.252+0000 7fec577fe640 1 -- 172.21.15.83:0/3981728111 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fec58099660 msgr2=0x7fec581c5430 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:25.256 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:25.252+0000 7fec577fe640 1 --2- 172.21.15.83:0/3981728111 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fec58099660 0x7fec581c5430 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:25.256 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:25.252+0000 7fec577fe640 1 -- 172.21.15.83:0/3981728111 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fec58129460 con 0x7fec58098d40 2024-09-14T03:40:25.256 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:25.252+0000 7fec56ffd640 1 --2- 172.21.15.83:0/3981728111 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fec58099660 0x7fec581c5430 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:40:25.256 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:25.252+0000 7fec577fe640 1 --2- 172.21.15.83:0/3981728111 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fec58098d40 0x7fec581c4ef0 secure :-1 s=READY pgs=36 cs=0 l=1 rev1=1 crypto rx=0x7fec48034040 tx=0x7fec48032e60 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:40:25.256 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:25.252+0000 7fec3ffff640 1 -- 172.21.15.83:0/3981728111 <== mon.1 v2:172.21.15.83:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fec48041020 con 0x7fec58098d40 2024-09-14T03:40:25.256 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:25.252+0000 7fec3ffff640 1 -- 172.21.15.83:0/3981728111 <== mon.1 v2:172.21.15.83:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fec48004040 con 0x7fec58098d40 2024-09-14T03:40:25.256 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:25.252+0000 7fec5dcd4640 1 -- 172.21.15.83:0/3981728111 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fec581c5970 con 0x7fec58098d40 2024-09-14T03:40:25.257 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:25.252+0000 7fec3ffff640 1 -- 172.21.15.83:0/3981728111 <== mon.1 v2:172.21.15.83:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fec48036040 con 0x7fec58098d40 2024-09-14T03:40:25.257 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:25.252+0000 7fec5dcd4640 1 -- 172.21.15.83:0/3981728111 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7fec580740f0 con 0x7fec58098d40 2024-09-14T03:40:25.257 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:25.253+0000 7fec5dcd4640 1 -- 172.21.15.83:0/3981728111 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fec580008d0 con 0x7fec58098d40 2024-09-14T03:40:25.257 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:25.253+0000 7fec3ffff640 1 -- 172.21.15.83:0/3981728111 <== mon.1 v2:172.21.15.83:3301/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7fec48044020 con 0x7fec58098d40 2024-09-14T03:40:25.257 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:25.254+0000 7fec3ffff640 1 -- 172.21.15.83:0/3981728111 <== mon.1 v2:172.21.15.83:3301/0 5 ==== osd_map(22..22 src has 1..22) ==== 3027+0+0 (secure 0 0 0) 0x7fec48089d20 con 0x7fec58098d40 2024-09-14T03:40:25.258 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:25.257+0000 7fec3ffff640 1 -- 172.21.15.83:0/3981728111 <== mon.1 v2:172.21.15.83:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+157506 (secure 0 0 0) 0x7fec4804b030 con 0x7fec58098d40 2024-09-14T03:40:25.414 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:25.412+0000 7fec5dcd4640 1 -- 172.21.15.83:0/3981728111 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_command({"prefix": "osd down", "ids": ["1"]} v 0) -- 0x7fec58099120 con 0x7fec58098d40 2024-09-14T03:40:25.414 INFO:tasks.ceph.mon.a.smithi083.stderr:2024-09-14T03:40:25.413+0000 7f14662ef640 -1 mon.a@0(leader).osd e22 definitely_dead 0 2024-09-14T03:40:26.066 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:26.065+0000 7fec3ffff640 1 -- 172.21.15.83:0/3981728111 <== mon.1 v2:172.21.15.83:3301/0 7 ==== mon_command_ack([{"prefix": "osd down", "ids": ["1"]}]=0 marked down osd.1. v23) ==== 89+0+0 (secure 0 0 0) 0x7fec48088dd0 con 0x7fec58098d40 2024-09-14T03:40:26.066 INFO:teuthology.orchestra.run.smithi083.stderr:marked down osd.1. 2024-09-14T03:40:26.069 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:26.067+0000 7fec5dcd4640 1 -- 172.21.15.83:0/3981728111 >> v1:172.21.15.83:6800/1803897559 conn(0x7fec440514e0 legacy=0x7fec440539a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:26.069 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:26.067+0000 7fec5dcd4640 1 -- 172.21.15.83:0/3981728111 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fec58098d40 msgr2=0x7fec581c4ef0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:26.069 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:26.067+0000 7fec5dcd4640 1 --2- 172.21.15.83:0/3981728111 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fec58098d40 0x7fec581c4ef0 secure :-1 s=READY pgs=36 cs=0 l=1 rev1=1 crypto rx=0x7fec48034040 tx=0x7fec48032e60 comp rx=0 tx=0).stop 2024-09-14T03:40:26.069 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:26.067+0000 7fec5dcd4640 1 -- 172.21.15.83:0/3981728111 shutdown_connections 2024-09-14T03:40:26.069 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:26.067+0000 7fec5dcd4640 1 --2- 172.21.15.83:0/3981728111 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fec5812f1c0 0x7fec58071520 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:26.069 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:26.067+0000 7fec5dcd4640 1 --2- 172.21.15.83:0/3981728111 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fec58099660 0x7fec581c5430 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:26.069 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:26.067+0000 7fec5dcd4640 1 --2- 172.21.15.83:0/3981728111 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fec58098d40 0x7fec581c4ef0 unknown :-1 s=CLOSED pgs=36 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:26.069 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:26.067+0000 7fec5dcd4640 1 -- 172.21.15.83:0/3981728111 >> 172.21.15.83:0/3981728111 conn(0x7fec580a1790 msgr2=0x7fec5809d230 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:26.070 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:26.067+0000 7fec5dcd4640 1 -- 172.21.15.83:0/3981728111 shutdown_connections 2024-09-14T03:40:26.070 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:26.067+0000 7fec5dcd4640 1 -- 172.21.15.83:0/3981728111 wait complete. 2024-09-14T03:40:26.080 INFO:tasks.ceph.osd.1:Restarting daemon 2024-09-14T03:40:26.080 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster ceph -i 1 2024-09-14T03:40:26.082 INFO:tasks.ceph.osd.1:Started 2024-09-14T03:40:26.083 DEBUG:tasks.ceph.osd.2:waiting for process to exit 2024-09-14T03:40:26.083 INFO:teuthology.orchestra.run:waiting for 300 2024-09-14T03:40:26.124 INFO:tasks.ceph.osd.2:Stopped 2024-09-14T03:40:26.124 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd down 2 2024-09-14T03:40:26.259 INFO:tasks.ceph.osd.0.smithi083.stderr:2024-09-14T03:40:26.257+0000 7fc628864640 -1 Falling back to public interface 2024-09-14T03:40:26.268 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:26.266+0000 7f38e21a9640 1 -- 172.21.15.83:0/2438434720 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f38dc0995e0 msgr2=0x7f38dc0999c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:26.268 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:26.266+0000 7f38e21a9640 1 --2- 172.21.15.83:0/2438434720 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f38dc0995e0 0x7f38dc0999c0 secure :-1 s=READY pgs=53 cs=0 l=1 rev1=1 crypto rx=0x7f38cc009860 tx=0x7f38cc02a420 comp rx=0 tx=0).stop 2024-09-14T03:40:26.268 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:26.266+0000 7f38e21a9640 1 -- 172.21.15.83:0/2438434720 shutdown_connections 2024-09-14T03:40:26.268 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:26.266+0000 7f38e21a9640 1 --2- 172.21.15.83:0/2438434720 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f38dc0716b0 0x7f38dc134780 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:26.268 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:26.266+0000 7f38e21a9640 1 --2- 172.21.15.83:0/2438434720 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f38dc0995e0 0x7f38dc0999c0 unknown :-1 s=CLOSED pgs=53 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:26.268 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:26.266+0000 7f38e21a9640 1 -- 172.21.15.83:0/2438434720 >> 172.21.15.83:0/2438434720 conn(0x7f38dc09d5d0 msgr2=0x7f38dc09d9b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:26.269 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:26.266+0000 7f38e21a9640 1 -- 172.21.15.83:0/2438434720 shutdown_connections 2024-09-14T03:40:26.269 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:26.266+0000 7f38e21a9640 1 -- 172.21.15.83:0/2438434720 wait complete. 2024-09-14T03:40:26.271 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:26.267+0000 7f38e21a9640 1 Processor -- start 2024-09-14T03:40:26.271 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:26.267+0000 7f38e21a9640 1 -- start start 2024-09-14T03:40:26.271 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:26.267+0000 7f38e21a9640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f38dc070b90 0x7f38dc122690 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:26.271 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:26.267+0000 7f38e21a9640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f38dc0716b0 0x7f38dc122bd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:26.271 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:26.268+0000 7f38e21a9640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f38dc0995e0 0x7f38dc123110 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:26.271 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:26.268+0000 7f38e21a9640 1 -- --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7f38dc135920 con 0x7f38dc0995e0 2024-09-14T03:40:26.271 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:26.268+0000 7f38e21a9640 1 -- --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7f38dc1357a0 con 0x7f38dc0716b0 2024-09-14T03:40:26.271 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:26.268+0000 7f38e21a9640 1 -- --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7f38dc135aa0 con 0x7f38dc070b90 2024-09-14T03:40:26.271 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:26.268+0000 7f38db7fe640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f38dc070b90 0x7f38dc122690 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:26.272 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:26.268+0000 7f38db7fe640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f38dc070b90 0x7f38dc122690 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3302/0 says I am v2:172.21.15.83:37914/0 (socket says 172.21.15.83:37914) 2024-09-14T03:40:26.272 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:26.268+0000 7f38db7fe640 1 -- 172.21.15.83:0/3701238967 learned_addr learned my addr 172.21.15.83:0/3701238967 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:40:26.272 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:26.268+0000 7f38d3fff640 1 --2- 172.21.15.83:0/3701238967 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f38dc0716b0 0x7f38dc122bd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:26.272 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:26.268+0000 7f38dbfff640 1 --2- 172.21.15.83:0/3701238967 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f38dc0995e0 0x7f38dc123110 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:26.272 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:26.268+0000 7f38db7fe640 1 -- 172.21.15.83:0/3701238967 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f38dc0716b0 msgr2=0x7f38dc122bd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:26.272 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:26.268+0000 7f38db7fe640 1 --2- 172.21.15.83:0/3701238967 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f38dc0716b0 0x7f38dc122bd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:26.272 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:26.268+0000 7f38db7fe640 1 -- 172.21.15.83:0/3701238967 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f38dc0995e0 msgr2=0x7f38dc123110 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:26.272 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:26.268+0000 7f38db7fe640 1 --2- 172.21.15.83:0/3701238967 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f38dc0995e0 0x7f38dc123110 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:26.272 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:26.268+0000 7f38db7fe640 1 -- 172.21.15.83:0/3701238967 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f38dc1248e0 con 0x7f38dc070b90 2024-09-14T03:40:26.272 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:26.268+0000 7f38dbfff640 1 --2- 172.21.15.83:0/3701238967 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f38dc0995e0 0x7f38dc123110 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:40:26.272 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:26.268+0000 7f38db7fe640 1 --2- 172.21.15.83:0/3701238967 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f38dc070b90 0x7f38dc122690 secure :-1 s=READY pgs=39 cs=0 l=1 rev1=1 crypto rx=0x7f38cc009980 tx=0x7f38cc032b90 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:40:26.272 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:26.268+0000 7f38d8ff9640 1 -- 172.21.15.83:0/3701238967 <== mon.2 v2:172.21.15.83:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f38cc040070 con 0x7f38dc070b90 2024-09-14T03:40:26.272 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:26.268+0000 7f38e21a9640 1 -- 172.21.15.83:0/3701238967 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f38dc073730 con 0x7f38dc070b90 2024-09-14T03:40:26.273 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:26.268+0000 7f38d8ff9640 1 -- 172.21.15.83:0/3701238967 <== mon.2 v2:172.21.15.83:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f38cc005310 con 0x7f38dc070b90 2024-09-14T03:40:26.273 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:26.268+0000 7f38d8ff9640 1 -- 172.21.15.83:0/3701238967 <== mon.2 v2:172.21.15.83:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f38cc0047a0 con 0x7f38dc070b90 2024-09-14T03:40:26.273 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:26.268+0000 7f38e21a9640 1 -- 172.21.15.83:0/3701238967 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7f38dc1347d0 con 0x7f38dc070b90 2024-09-14T03:40:26.273 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:26.269+0000 7f38e21a9640 1 -- 172.21.15.83:0/3701238967 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f38dc134a30 con 0x7f38dc070b90 2024-09-14T03:40:26.273 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:26.269+0000 7f38d8ff9640 1 -- 172.21.15.83:0/3701238967 <== mon.2 v2:172.21.15.83:3302/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7f38cc039040 con 0x7f38dc070b90 2024-09-14T03:40:26.273 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:26.270+0000 7f38d8ff9640 1 -- 172.21.15.83:0/3701238967 <== mon.2 v2:172.21.15.83:3302/0 5 ==== osd_map(23..23 src has 1..23) ==== 3027+0+0 (secure 0 0 0) 0x7f38cc08a0e0 con 0x7f38dc070b90 2024-09-14T03:40:26.275 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:26.273+0000 7f38d8ff9640 1 -- 172.21.15.83:0/3701238967 <== mon.2 v2:172.21.15.83:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+157506 (secure 0 0 0) 0x7f38cc089190 con 0x7f38dc070b90 2024-09-14T03:40:26.431 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:26.429+0000 7f38e21a9640 1 -- 172.21.15.83:0/3701238967 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_command({"prefix": "osd down", "ids": ["2"]} v 0) -- 0x7f38dc0999c0 con 0x7f38dc070b90 2024-09-14T03:40:26.431 INFO:tasks.ceph.mon.a.smithi083.stderr:2024-09-14T03:40:26.430+0000 7f14662ef640 -1 mon.a@0(leader).osd e23 definitely_dead 0 2024-09-14T03:40:26.995 INFO:tasks.ceph.osd.1.smithi083.stderr:2024-09-14T03:40:26.993+0000 7fa3cfa0b640 -1 Falling back to public interface 2024-09-14T03:40:27.068 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:27.067+0000 7f38d8ff9640 1 -- 172.21.15.83:0/3701238967 <== mon.2 v2:172.21.15.83:3302/0 7 ==== mon_command_ack([{"prefix": "osd down", "ids": ["2"]}]=0 marked down osd.2. v24) ==== 89+0+0 (secure 0 0 0) 0x7f38cc03fd90 con 0x7f38dc070b90 2024-09-14T03:40:27.069 INFO:teuthology.orchestra.run.smithi083.stderr:marked down osd.2. 2024-09-14T03:40:27.071 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:27.069+0000 7f38e21a9640 1 -- 172.21.15.83:0/3701238967 >> v1:172.21.15.83:6800/1803897559 conn(0x7f38bc0514b0 legacy=0x7f38bc053970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:27.071 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:27.069+0000 7f38e21a9640 1 -- 172.21.15.83:0/3701238967 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f38dc070b90 msgr2=0x7f38dc122690 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:27.071 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:27.069+0000 7f38e21a9640 1 --2- 172.21.15.83:0/3701238967 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f38dc070b90 0x7f38dc122690 secure :-1 s=READY pgs=39 cs=0 l=1 rev1=1 crypto rx=0x7f38cc009980 tx=0x7f38cc032b90 comp rx=0 tx=0).stop 2024-09-14T03:40:27.071 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:27.069+0000 7f38e21a9640 1 -- 172.21.15.83:0/3701238967 shutdown_connections 2024-09-14T03:40:27.071 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:27.069+0000 7f38e21a9640 1 --2- 172.21.15.83:0/3701238967 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f38dc0995e0 0x7f38dc123110 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:27.071 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:27.069+0000 7f38e21a9640 1 --2- 172.21.15.83:0/3701238967 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f38dc0716b0 0x7f38dc122bd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:27.072 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:27.069+0000 7f38e21a9640 1 --2- 172.21.15.83:0/3701238967 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f38dc070b90 0x7f38dc122690 unknown :-1 s=CLOSED pgs=39 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:27.072 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:27.069+0000 7f38e21a9640 1 -- 172.21.15.83:0/3701238967 >> 172.21.15.83:0/3701238967 conn(0x7f38dc09d5d0 msgr2=0x7f38dc06fa10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:27.072 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:27.070+0000 7f38e21a9640 1 -- 172.21.15.83:0/3701238967 shutdown_connections 2024-09-14T03:40:27.072 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:27.070+0000 7f38e21a9640 1 -- 172.21.15.83:0/3701238967 wait complete. 2024-09-14T03:40:27.084 INFO:tasks.ceph.osd.2:Restarting daemon 2024-09-14T03:40:27.084 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster ceph -i 2 2024-09-14T03:40:27.086 INFO:tasks.ceph.osd.2:Started 2024-09-14T03:40:27.087 DEBUG:tasks.ceph.osd.3:waiting for process to exit 2024-09-14T03:40:27.087 INFO:teuthology.orchestra.run:waiting for 300 2024-09-14T03:40:27.128 INFO:tasks.ceph.osd.3:Stopped 2024-09-14T03:40:27.128 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd down 3 2024-09-14T03:40:27.268 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:27.266+0000 7fe79aa8c640 1 -- 172.21.15.83:0/1164513878 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fe79412b490 msgr2=0x7fe79412b870 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:27.268 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:27.266+0000 7fe79aa8c640 1 --2- 172.21.15.83:0/1164513878 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fe79412b490 0x7fe79412b870 secure :-1 s=READY pgs=40 cs=0 l=1 rev1=1 crypto rx=0x7fe78800a970 tx=0x7fe78802b7f0 comp rx=0 tx=0).stop 2024-09-14T03:40:27.268 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:27.266+0000 7fe79aa8c640 1 -- 172.21.15.83:0/1164513878 shutdown_connections 2024-09-14T03:40:27.268 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:27.266+0000 7fe79aa8c640 1 --2- 172.21.15.83:0/1164513878 >> v2:172.21.15.83:3300/0 conn(0x7fe794095570 0x7fe7940959e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:27.268 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:27.266+0000 7fe79aa8c640 1 --2- 172.21.15.83:0/1164513878 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fe79412b490 0x7fe79412b870 unknown :-1 s=CLOSED pgs=40 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:27.268 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:27.266+0000 7fe79aa8c640 1 -- 172.21.15.83:0/1164513878 >> 172.21.15.83:0/1164513878 conn(0x7fe79409afd0 msgr2=0x7fe79409b3e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:27.268 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:27.266+0000 7fe79aa8c640 1 -- 172.21.15.83:0/1164513878 shutdown_connections 2024-09-14T03:40:27.268 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:27.266+0000 7fe79aa8c640 1 -- 172.21.15.83:0/1164513878 wait complete. 2024-09-14T03:40:27.268 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:27.267+0000 7fe79aa8c640 1 Processor -- start 2024-09-14T03:40:27.269 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:27.267+0000 7fe79aa8c640 1 -- start start 2024-09-14T03:40:27.271 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:27.267+0000 7fe79aa8c640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fe794095570 0x7fe7941c2be0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:27.271 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:27.267+0000 7fe79aa8c640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fe79412b490 0x7fe7941c3120 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:27.271 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:27.267+0000 7fe79aa8c640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fe79412bdb0 0x7fe79412c440 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:27.271 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:27.267+0000 7fe79aa8c640 1 -- --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7fe794133600 con 0x7fe79412bdb0 2024-09-14T03:40:27.271 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:27.267+0000 7fe79aa8c640 1 -- --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7fe794133480 con 0x7fe794095570 2024-09-14T03:40:27.271 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:27.267+0000 7fe79aa8c640 1 -- --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7fe794133780 con 0x7fe79412b490 2024-09-14T03:40:27.271 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:27.267+0000 7fe793fff640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fe79412b490 0x7fe7941c3120 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:27.271 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:27.267+0000 7fe799029640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fe79412bdb0 0x7fe79412c440 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:27.271 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:27.267+0000 7fe798828640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fe794095570 0x7fe7941c2be0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:27.271 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:27.267+0000 7fe799029640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fe79412bdb0 0x7fe79412c440 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3300/0 says I am v2:172.21.15.83:40090/0 (socket says 172.21.15.83:40090) 2024-09-14T03:40:27.271 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:27.267+0000 7fe799029640 1 -- 172.21.15.83:0/240282275 learned_addr learned my addr 172.21.15.83:0/240282275 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:40:27.271 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:27.268+0000 7fe793fff640 1 -- 172.21.15.83:0/240282275 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fe794095570 msgr2=0x7fe7941c2be0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:27.271 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:27.268+0000 7fe793fff640 1 --2- 172.21.15.83:0/240282275 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fe794095570 0x7fe7941c2be0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:27.272 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:27.268+0000 7fe793fff640 1 -- 172.21.15.83:0/240282275 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fe79412bdb0 msgr2=0x7fe79412c440 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:27.272 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:27.268+0000 7fe793fff640 1 --2- 172.21.15.83:0/240282275 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fe79412bdb0 0x7fe79412c440 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:27.272 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:27.268+0000 7fe793fff640 1 -- 172.21.15.83:0/240282275 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fe79412ea30 con 0x7fe79412b490 2024-09-14T03:40:27.272 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:27.268+0000 7fe793fff640 1 --2- 172.21.15.83:0/240282275 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fe79412b490 0x7fe7941c3120 secure :-1 s=READY pgs=41 cs=0 l=1 rev1=1 crypto rx=0x7fe78000c430 tx=0x7fe78000c8f0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:40:27.272 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:27.268+0000 7fe7917fa640 1 -- 172.21.15.83:0/240282275 <== mon.2 v2:172.21.15.83:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe780017070 con 0x7fe79412b490 2024-09-14T03:40:27.272 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:27.268+0000 7fe7917fa640 1 -- 172.21.15.83:0/240282275 <== mon.2 v2:172.21.15.83:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fe7800053c0 con 0x7fe79412b490 2024-09-14T03:40:27.272 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:27.268+0000 7fe79aa8c640 1 -- 172.21.15.83:0/240282275 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fe79412e5c0 con 0x7fe79412b490 2024-09-14T03:40:27.272 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:27.268+0000 7fe7917fa640 1 -- 172.21.15.83:0/240282275 <== mon.2 v2:172.21.15.83:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe780003850 con 0x7fe79412b490 2024-09-14T03:40:27.272 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:27.268+0000 7fe79aa8c640 1 -- 172.21.15.83:0/240282275 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7fe7941c3660 con 0x7fe79412b490 2024-09-14T03:40:27.272 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:27.269+0000 7fe79aa8c640 1 -- 172.21.15.83:0/240282275 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fe7940959e0 con 0x7fe79412b490 2024-09-14T03:40:27.272 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:27.269+0000 7fe7917fa640 1 -- 172.21.15.83:0/240282275 <== mon.2 v2:172.21.15.83:3302/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7fe78001a070 con 0x7fe79412b490 2024-09-14T03:40:27.272 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:27.269+0000 7fe7917fa640 1 -- 172.21.15.83:0/240282275 <== mon.2 v2:172.21.15.83:3302/0 5 ==== osd_map(24..24 src has 1..24) ==== 3027+0+0 (secure 0 0 0) 0x7fe780068360 con 0x7fe79412b490 2024-09-14T03:40:27.274 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:27.273+0000 7fe7917fa640 1 -- 172.21.15.83:0/240282275 <== mon.2 v2:172.21.15.83:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+157506 (secure 0 0 0) 0x7fe79412ea30 con 0x7fe79412b490 2024-09-14T03:40:27.432 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:27.431+0000 7fe79aa8c640 1 -- 172.21.15.83:0/240282275 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_command({"prefix": "osd down", "ids": ["3"]} v 0) -- 0x7fe7940699c0 con 0x7fe79412b490 2024-09-14T03:40:27.433 INFO:tasks.ceph.mon.a.smithi083.stderr:2024-09-14T03:40:27.431+0000 7f14662ef640 -1 mon.a@0(leader).osd e24 definitely_dead 0 2024-09-14T03:40:28.069 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:28.068+0000 7fe7917fa640 1 -- 172.21.15.83:0/240282275 <== mon.2 v2:172.21.15.83:3302/0 7 ==== mon_command_ack([{"prefix": "osd down", "ids": ["3"]}]=0 marked down osd.3. v25) ==== 89+0+0 (secure 0 0 0) 0x7fe780067410 con 0x7fe79412b490 2024-09-14T03:40:28.070 INFO:teuthology.orchestra.run.smithi083.stderr:marked down osd.3. 2024-09-14T03:40:28.072 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:28.070+0000 7fe79aa8c640 1 -- 172.21.15.83:0/240282275 >> v1:172.21.15.83:6800/1803897559 conn(0x7fe7780514b0 legacy=0x7fe778053970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:28.072 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:28.070+0000 7fe79aa8c640 1 -- 172.21.15.83:0/240282275 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fe79412b490 msgr2=0x7fe7941c3120 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:28.072 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:28.070+0000 7fe79aa8c640 1 --2- 172.21.15.83:0/240282275 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fe79412b490 0x7fe7941c3120 secure :-1 s=READY pgs=41 cs=0 l=1 rev1=1 crypto rx=0x7fe78000c430 tx=0x7fe78000c8f0 comp rx=0 tx=0).stop 2024-09-14T03:40:28.072 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:28.070+0000 7fe79aa8c640 1 -- 172.21.15.83:0/240282275 shutdown_connections 2024-09-14T03:40:28.072 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:28.070+0000 7fe79aa8c640 1 --2- 172.21.15.83:0/240282275 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fe79412bdb0 0x7fe79412c440 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:28.072 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:28.070+0000 7fe79aa8c640 1 --2- 172.21.15.83:0/240282275 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fe79412b490 0x7fe7941c3120 unknown :-1 s=CLOSED pgs=41 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:28.072 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:28.070+0000 7fe79aa8c640 1 --2- 172.21.15.83:0/240282275 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fe794095570 0x7fe7941c2be0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:28.072 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:28.070+0000 7fe79aa8c640 1 -- 172.21.15.83:0/240282275 >> 172.21.15.83:0/240282275 conn(0x7fe79409afd0 msgr2=0x7fe7940901a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:28.072 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:28.070+0000 7fe79aa8c640 1 -- 172.21.15.83:0/240282275 shutdown_connections 2024-09-14T03:40:28.072 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:28.070+0000 7fe79aa8c640 1 -- 172.21.15.83:0/240282275 wait complete. 2024-09-14T03:40:28.084 INFO:tasks.ceph.osd.3:Restarting daemon 2024-09-14T03:40:28.084 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster ceph -i 3 2024-09-14T03:40:28.086 INFO:tasks.ceph.osd.3:Started 2024-09-14T03:40:28.086 INFO:tasks.ceph.ceph_manager.ceph:waiting for all up 2024-09-14T03:40:28.086 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-09-14T03:40:28.188 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:28.186+0000 7fdb74daa640 1 -- 172.21.15.83:0/3262233183 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fdb7012eff0 msgr2=0x7fdb7012f460 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:28.188 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:28.186+0000 7fdb74daa640 1 --2- 172.21.15.83:0/3262233183 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fdb7012eff0 0x7fdb7012f460 secure :-1 s=READY pgs=39 cs=0 l=1 rev1=1 crypto rx=0x7fdb68005fd0 tx=0x7fdb6802a970 comp rx=0 tx=0).stop 2024-09-14T03:40:28.188 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:28.186+0000 7fdb74daa640 1 -- 172.21.15.83:0/3262233183 shutdown_connections 2024-09-14T03:40:28.188 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:28.186+0000 7fdb74daa640 1 --2- 172.21.15.83:0/3262233183 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fdb7012eff0 0x7fdb7012f460 unknown :-1 s=CLOSED pgs=39 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:28.188 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:28.186+0000 7fdb74daa640 1 --2- 172.21.15.83:0/3262233183 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fdb700999a0 0x7fdb7012eab0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:28.189 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:28.186+0000 7fdb74daa640 1 --2- 172.21.15.83:0/3262233183 >> v2:172.21.15.83:3300/0 conn(0x7fdb70099080 0x7fdb70099460 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:28.189 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:28.186+0000 7fdb74daa640 1 -- 172.21.15.83:0/3262233183 >> 172.21.15.83:0/3262233183 conn(0x7fdb700a1800 msgr2=0x7fdb700a1c10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:28.189 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:28.186+0000 7fdb74daa640 1 -- 172.21.15.83:0/3262233183 shutdown_connections 2024-09-14T03:40:28.189 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:28.186+0000 7fdb74daa640 1 -- 172.21.15.83:0/3262233183 wait complete. 2024-09-14T03:40:28.191 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:28.187+0000 7fdb74daa640 1 Processor -- start 2024-09-14T03:40:28.192 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:28.187+0000 7fdb74daa640 1 -- start start 2024-09-14T03:40:28.192 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:28.187+0000 7fdb74daa640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fdb70099080 0x7fdb701c4dc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:28.192 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:28.187+0000 7fdb74daa640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fdb700999a0 0x7fdb701c5300 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:28.192 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:28.187+0000 7fdb74daa640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fdb7012eff0 0x7fdb70132b50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:28.192 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:28.187+0000 7fdb74daa640 1 -- --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7fdb701366c0 con 0x7fdb700999a0 2024-09-14T03:40:28.192 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:28.187+0000 7fdb74daa640 1 -- --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7fdb70136540 con 0x7fdb70099080 2024-09-14T03:40:28.192 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:28.187+0000 7fdb74daa640 1 -- --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7fdb70136840 con 0x7fdb7012eff0 2024-09-14T03:40:28.192 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:28.188+0000 7fdb6e59c640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fdb70099080 0x7fdb701c4dc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:28.192 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:28.188+0000 7fdb6dd9b640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fdb700999a0 0x7fdb701c5300 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:28.192 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:28.188+0000 7fdb6dd9b640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fdb700999a0 0x7fdb701c5300 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3300/0 says I am v2:172.21.15.83:40124/0 (socket says 172.21.15.83:40124) 2024-09-14T03:40:28.192 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:28.188+0000 7fdb6dd9b640 1 -- 172.21.15.83:0/2729883822 learned_addr learned my addr 172.21.15.83:0/2729883822 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:40:28.193 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:28.188+0000 7fdb6e59c640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fdb70099080 0x7fdb701c4dc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3301/0 says I am v2:172.21.15.83:35832/0 (socket says 172.21.15.83:35832) 2024-09-14T03:40:28.193 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:28.188+0000 7fdb6ed9d640 1 --2- 172.21.15.83:0/2729883822 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fdb7012eff0 0x7fdb70132b50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:28.193 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:28.188+0000 7fdb6e59c640 1 -- 172.21.15.83:0/2729883822 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fdb7012eff0 msgr2=0x7fdb70132b50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:28.193 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:28.188+0000 7fdb6e59c640 1 --2- 172.21.15.83:0/2729883822 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fdb7012eff0 0x7fdb70132b50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:28.193 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:28.188+0000 7fdb6e59c640 1 -- 172.21.15.83:0/2729883822 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fdb700999a0 msgr2=0x7fdb701c5300 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:28.193 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:28.188+0000 7fdb6e59c640 1 --2- 172.21.15.83:0/2729883822 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fdb700999a0 0x7fdb701c5300 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:28.193 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:28.188+0000 7fdb6e59c640 1 -- 172.21.15.83:0/2729883822 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fdb701294d0 con 0x7fdb70099080 2024-09-14T03:40:28.193 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:28.188+0000 7fdb6dd9b640 1 --2- 172.21.15.83:0/2729883822 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fdb700999a0 0x7fdb701c5300 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:40:28.193 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:28.188+0000 7fdb6e59c640 1 --2- 172.21.15.83:0/2729883822 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fdb70099080 0x7fdb701c4dc0 secure :-1 s=READY pgs=40 cs=0 l=1 rev1=1 crypto rx=0x7fdb6000e9f0 tx=0x7fdb6000eeb0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:40:28.193 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:28.188+0000 7fdb5effd640 1 -- 172.21.15.83:0/2729883822 <== mon.1 v2:172.21.15.83:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fdb6000cd80 con 0x7fdb70099080 2024-09-14T03:40:28.193 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:28.188+0000 7fdb5effd640 1 -- 172.21.15.83:0/2729883822 <== mon.1 v2:172.21.15.83:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fdb60004020 con 0x7fdb70099080 2024-09-14T03:40:28.194 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:28.188+0000 7fdb74daa640 1 -- 172.21.15.83:0/2729883822 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fdb70135270 con 0x7fdb70099080 2024-09-14T03:40:28.194 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:28.188+0000 7fdb5effd640 1 -- 172.21.15.83:0/2729883822 <== mon.1 v2:172.21.15.83:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fdb60004450 con 0x7fdb70099080 2024-09-14T03:40:28.194 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:28.188+0000 7fdb74daa640 1 -- 172.21.15.83:0/2729883822 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7fdb701c5840 con 0x7fdb70099080 2024-09-14T03:40:28.194 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:28.189+0000 7fdb74daa640 1 -- 172.21.15.83:0/2729883822 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fdb7012eab0 con 0x7fdb70099080 2024-09-14T03:40:28.194 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:28.189+0000 7fdb5effd640 1 -- 172.21.15.83:0/2729883822 <== mon.1 v2:172.21.15.83:3301/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7fdb600035f0 con 0x7fdb70099080 2024-09-14T03:40:28.194 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:28.190+0000 7fdb5effd640 1 -- 172.21.15.83:0/2729883822 <== mon.1 v2:172.21.15.83:3301/0 5 ==== osd_map(25..25 src has 1..25) ==== 3027+0+0 (secure 0 0 0) 0x7fdb600698b0 con 0x7fdb70099080 2024-09-14T03:40:28.194 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:28.193+0000 7fdb5effd640 1 -- 172.21.15.83:0/2729883822 <== mon.1 v2:172.21.15.83:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+157506 (secure 0 0 0) 0x7fdb60068960 con 0x7fdb70099080 2024-09-14T03:40:28.343 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:28.341+0000 7fdb74daa640 1 -- 172.21.15.83:0/2729883822 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) -- 0x7fdb700698c0 con 0x7fdb70099080 2024-09-14T03:40:28.343 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:28.342+0000 7fdb5effd640 1 -- 172.21.15.83:0/2729883822 <== mon.1 v2:172.21.15.83:3301/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v25) ==== 74+0+8657 (secure 0 0 0) 0x7fdb60009310 con 0x7fdb70099080 2024-09-14T03:40:28.343 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:40:28.344 INFO:teuthology.orchestra.run.smithi083.stdout:{"epoch":25,"fsid":"765298ef-df4c-4f28-af02-b45f60264ff9","created":"2024-09-14T03:38:43.954952+0000","modified":"2024-09-14T03:40:28.062867+0000","last_up_change":"2024-09-14T03:40:28.062867+0000","last_in_change":"2024-09-14T03:38:46.006167+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":4,"require_min_compat_client":"squid","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-14T03:38:56.922548+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"14","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-14T03:39:00.433011+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":18,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"ff477614-343d-4d97-9df5-94933f9b904d","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":0,"down_at":22,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6805","nonce":2271402019}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6806","nonce":2271402019}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6808","nonce":2271402019}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6807","nonce":2271402019}]},"public_addr":"172.21.15.83:6805/2271402019","cluster_addr":"172.21.15.83:6806/2271402019","heartbeat_back_addr":"172.21.15.83:6808/2271402019","heartbeat_front_addr":"172.21.15.83:6807/2271402019","state":["exists"]},{"osd":1,"uuid":"562ec9a0-3f91-4a29-896d-94340e9fa36f","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":11,"up_thru":15,"down_at":23,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6801","nonce":2493296278}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6802","nonce":2493296278}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6804","nonce":2493296278}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6803","nonce":2493296278}]},"public_addr":"172.21.15.83:6801/2493296278","cluster_addr":"172.21.15.83:6802/2493296278","heartbeat_back_addr":"172.21.15.83:6804/2493296278","heartbeat_front_addr":"172.21.15.83:6803/2493296278","state":["exists"]},{"osd":2,"uuid":"8c242dda-3308-4804-b2be-9a1737c42cf6","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":11,"up_thru":15,"down_at":24,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6810","nonce":1553686396}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6812","nonce":1553686396}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6816","nonce":1553686396}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6814","nonce":1553686396}]},"public_addr":"172.21.15.83:6810/1553686396","cluster_addr":"172.21.15.83:6812/1553686396","heartbeat_back_addr":"172.21.15.83:6816/1553686396","heartbeat_front_addr":"172.21.15.83:6814/1553686396","state":["exists"]},{"osd":3,"uuid":"c4dd2c36-3c86-4b56-86fb-fede641a436c","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":11,"up_thru":24,"down_at":25,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6809","nonce":2350261690}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6811","nonce":2350261690}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6815","nonce":2350261690}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6813","nonce":2350261690}]},"public_addr":"172.21.15.83:6809/2350261690","cluster_addr":"172.21.15.83:6811/2350261690","heartbeat_back_addr":"172.21.15.83:6815/2350261690","heartbeat_front_addr":"172.21.15.83:6813/2350261690","state":["exists"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:38:53.047098+0000","dead_epoch":22},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:38:53.354351+0000","dead_epoch":23},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:38:53.422284+0000","dead_epoch":24},{"osd":3,"down_stamp":"2024-09-14T03:40:28.062867+0000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:38:53.620657+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.83:0/293392365":"2024-09-15T03:40:20.028874+0000","172.21.15.83:0/4246762177":"2024-09-15T03:40:20.028874+0000","172.21.15.83:0/3623382676":"2024-09-15T03:40:20.028874+0000","172.21.15.83:6800/1934165865":"2024-09-15T03:40:20.028874+0000","172.21.15.83:0/4208089382":"2024-09-15T03:40:20.028874+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-14T03:40:28.345 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:28.343+0000 7fdb74daa640 1 -- 172.21.15.83:0/2729883822 >> v1:172.21.15.83:6800/1803897559 conn(0x7fdb500514b0 legacy=0x7fdb50053970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:28.345 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:28.343+0000 7fdb74daa640 1 -- 172.21.15.83:0/2729883822 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fdb70099080 msgr2=0x7fdb701c4dc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:28.345 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:28.343+0000 7fdb74daa640 1 --2- 172.21.15.83:0/2729883822 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fdb70099080 0x7fdb701c4dc0 secure :-1 s=READY pgs=40 cs=0 l=1 rev1=1 crypto rx=0x7fdb6000e9f0 tx=0x7fdb6000eeb0 comp rx=0 tx=0).stop 2024-09-14T03:40:28.346 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:28.344+0000 7fdb74daa640 1 -- 172.21.15.83:0/2729883822 shutdown_connections 2024-09-14T03:40:28.346 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:28.344+0000 7fdb74daa640 1 --2- 172.21.15.83:0/2729883822 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fdb7012eff0 0x7fdb70132b50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:28.346 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:28.344+0000 7fdb74daa640 1 --2- 172.21.15.83:0/2729883822 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fdb700999a0 0x7fdb701c5300 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:28.346 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:28.344+0000 7fdb74daa640 1 --2- 172.21.15.83:0/2729883822 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fdb70099080 0x7fdb701c4dc0 unknown :-1 s=CLOSED pgs=40 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:28.346 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:28.344+0000 7fdb74daa640 1 -- 172.21.15.83:0/2729883822 >> 172.21.15.83:0/2729883822 conn(0x7fdb700a1800 msgr2=0x7fdb7012be60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:28.346 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:28.344+0000 7fdb74daa640 1 -- 172.21.15.83:0/2729883822 shutdown_connections 2024-09-14T03:40:28.346 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:28.344+0000 7fdb74daa640 1 -- 172.21.15.83:0/2729883822 wait complete. 2024-09-14T03:40:28.769 INFO:tasks.ceph.osd.2.smithi083.stderr:2024-09-14T03:40:28.767+0000 7fd359417640 -1 Falling back to public interface 2024-09-14T03:40:29.011 INFO:tasks.ceph.osd.3.smithi083.stderr:2024-09-14T03:40:29.009+0000 7f5e6bf20640 -1 Falling back to public interface 2024-09-14T03:40:30.316 INFO:tasks.ceph.osd.0.smithi083.stderr:2024-09-14T03:40:30.314+0000 7fc628864640 -1 osd.0 21 log_to_monitors true 2024-09-14T03:40:30.774 INFO:tasks.ceph.osd.1.smithi083.stderr:2024-09-14T03:40:30.772+0000 7fa3cfa0b640 -1 osd.1 21 log_to_monitors true 2024-09-14T03:40:31.090 INFO:tasks.ceph.osd.0.smithi083.stderr:2024-09-14T03:40:31.089+0000 7fc61c613640 -1 osd.0 21 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-14T03:40:31.090 INFO:tasks.ceph.osd.1.smithi083.stderr:2024-09-14T03:40:31.089+0000 7fa3c30b7640 -1 osd.1 21 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-14T03:40:31.357 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-09-14T03:40:31.461 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:31.460+0000 7fab0b80f640 1 -- 172.21.15.83:0/3248451888 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fab04099080 msgr2=0x7fab04099460 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:31.462 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:31.460+0000 7fab0b80f640 1 --2- 172.21.15.83:0/3248451888 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fab04099080 0x7fab04099460 secure :-1 s=READY pgs=59 cs=0 l=1 rev1=1 crypto rx=0x7fab00005fd0 tx=0x7fab0002a4a0 comp rx=0 tx=0).stop 2024-09-14T03:40:31.462 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:31.460+0000 7fab0b80f640 1 -- 172.21.15.83:0/3248451888 shutdown_connections 2024-09-14T03:40:31.462 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:31.460+0000 7fab0b80f640 1 --2- 172.21.15.83:0/3248451888 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fab040999a0 0x7fab0412eab0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:31.462 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:31.460+0000 7fab0b80f640 1 --2- 172.21.15.83:0/3248451888 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fab0412eff0 0x7fab0412f460 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:31.462 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:31.460+0000 7fab0b80f640 1 --2- 172.21.15.83:0/3248451888 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fab04099080 0x7fab04099460 unknown :-1 s=CLOSED pgs=59 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:31.462 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:31.460+0000 7fab0b80f640 1 -- 172.21.15.83:0/3248451888 >> 172.21.15.83:0/3248451888 conn(0x7fab040a1800 msgr2=0x7fab040a1c10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:31.462 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:31.460+0000 7fab0b80f640 1 -- 172.21.15.83:0/3248451888 shutdown_connections 2024-09-14T03:40:31.462 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:31.460+0000 7fab0b80f640 1 -- 172.21.15.83:0/3248451888 wait complete. 2024-09-14T03:40:31.464 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:31.460+0000 7fab0b80f640 1 Processor -- start 2024-09-14T03:40:31.464 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:31.461+0000 7fab0b80f640 1 -- start start 2024-09-14T03:40:31.464 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:31.461+0000 7fab0b80f640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fab04099080 0x7fab041c4b00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:31.464 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:31.461+0000 7fab0b80f640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fab040999a0 0x7fab041c5040 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:31.464 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:31.461+0000 7fab0b80f640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fab0412eff0 0x7fab041c5580 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:31.464 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:31.461+0000 7fab0b80f640 1 -- --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7fab04136670 con 0x7fab04099080 2024-09-14T03:40:31.465 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:31.461+0000 7fab0b80f640 1 -- --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7fab041364f0 con 0x7fab0412eff0 2024-09-14T03:40:31.465 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:31.461+0000 7fab0b80f640 1 -- --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7fab041367f0 con 0x7fab040999a0 2024-09-14T03:40:31.465 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:31.461+0000 7fab095ab640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fab04099080 0x7fab041c4b00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:31.465 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:31.461+0000 7fab08daa640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fab040999a0 0x7fab041c5040 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:31.465 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:31.461+0000 7fab09dac640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fab0412eff0 0x7fab041c5580 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:31.465 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:31.461+0000 7fab095ab640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fab04099080 0x7fab041c4b00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3300/0 says I am v2:172.21.15.83:40150/0 (socket says 172.21.15.83:40150) 2024-09-14T03:40:31.465 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:31.461+0000 7fab095ab640 1 -- 172.21.15.83:0/955378341 learned_addr learned my addr 172.21.15.83:0/955378341 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:40:31.465 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:31.461+0000 7fab09dac640 1 -- 172.21.15.83:0/955378341 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fab040999a0 msgr2=0x7fab041c5040 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:31.465 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:31.461+0000 7fab09dac640 1 --2- 172.21.15.83:0/955378341 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fab040999a0 0x7fab041c5040 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:31.465 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:31.461+0000 7fab09dac640 1 -- 172.21.15.83:0/955378341 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fab04099080 msgr2=0x7fab041c4b00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:31.465 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:31.461+0000 7fab09dac640 1 --2- 172.21.15.83:0/955378341 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fab04099080 0x7fab041c4b00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:31.465 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:31.461+0000 7fab09dac640 1 -- 172.21.15.83:0/955378341 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fab04073440 con 0x7fab0412eff0 2024-09-14T03:40:31.465 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:31.461+0000 7fab09dac640 1 --2- 172.21.15.83:0/955378341 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fab0412eff0 0x7fab041c5580 secure :-1 s=READY pgs=43 cs=0 l=1 rev1=1 crypto rx=0x7faafc00ec50 tx=0x7faafc00c6a0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:40:31.466 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:31.462+0000 7faaf5ffb640 1 -- 172.21.15.83:0/955378341 <== mon.1 v2:172.21.15.83:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7faafc003b70 con 0x7fab0412eff0 2024-09-14T03:40:31.466 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:31.462+0000 7faaf5ffb640 1 -- 172.21.15.83:0/955378341 <== mon.1 v2:172.21.15.83:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7faafc003d10 con 0x7fab0412eff0 2024-09-14T03:40:31.466 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:31.462+0000 7faaf5ffb640 1 -- 172.21.15.83:0/955378341 <== mon.1 v2:172.21.15.83:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7faafc004280 con 0x7fab0412eff0 2024-09-14T03:40:31.466 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:31.462+0000 7fab0b80f640 1 -- 172.21.15.83:0/955378341 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fab041294d0 con 0x7fab0412eff0 2024-09-14T03:40:31.466 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:31.462+0000 7fab0b80f640 1 -- 172.21.15.83:0/955378341 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7fab04071630 con 0x7fab0412eff0 2024-09-14T03:40:31.466 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:31.463+0000 7fab0b80f640 1 -- 172.21.15.83:0/955378341 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fab040008d0 con 0x7fab0412eff0 2024-09-14T03:40:31.466 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:31.463+0000 7faaf5ffb640 1 -- 172.21.15.83:0/955378341 <== mon.1 v2:172.21.15.83:3301/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7faafc004420 con 0x7fab0412eff0 2024-09-14T03:40:31.466 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:31.463+0000 7faaf5ffb640 1 -- 172.21.15.83:0/955378341 <== mon.1 v2:172.21.15.83:3301/0 5 ==== osd_map(26..26 src has 1..26) ==== 3027+0+0 (secure 0 0 0) 0x7faafc069bf0 con 0x7fab0412eff0 2024-09-14T03:40:31.468 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:31.467+0000 7faaf5ffb640 1 -- 172.21.15.83:0/955378341 <== mon.1 v2:172.21.15.83:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+157506 (secure 0 0 0) 0x7faafc068ca0 con 0x7fab0412eff0 2024-09-14T03:40:31.616 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:31.615+0000 7fab0b80f640 1 -- 172.21.15.83:0/955378341 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) -- 0x7fab0412fdd0 con 0x7fab0412eff0 2024-09-14T03:40:31.617 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:31.616+0000 7faaf5ffb640 1 -- 172.21.15.83:0/955378341 <== mon.1 v2:172.21.15.83:3301/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v26) ==== 74+0+8657 (secure 0 0 0) 0x7faafc004710 con 0x7fab0412eff0 2024-09-14T03:40:31.617 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:40:31.617 INFO:teuthology.orchestra.run.smithi083.stdout:{"epoch":26,"fsid":"765298ef-df4c-4f28-af02-b45f60264ff9","created":"2024-09-14T03:38:43.954952+0000","modified":"2024-09-14T03:40:31.080537+0000","last_up_change":"2024-09-14T03:40:28.062867+0000","last_in_change":"2024-09-14T03:38:46.006167+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":6,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":4,"require_min_compat_client":"squid","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-14T03:38:56.922548+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"14","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-14T03:39:00.433011+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":18,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"ff477614-343d-4d97-9df5-94933f9b904d","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":0,"down_at":22,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6805","nonce":2271402019}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6806","nonce":2271402019}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6808","nonce":2271402019}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6807","nonce":2271402019}]},"public_addr":"172.21.15.83:6805/2271402019","cluster_addr":"172.21.15.83:6806/2271402019","heartbeat_back_addr":"172.21.15.83:6808/2271402019","heartbeat_front_addr":"172.21.15.83:6807/2271402019","state":["exists"]},{"osd":1,"uuid":"562ec9a0-3f91-4a29-896d-94340e9fa36f","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":11,"up_thru":15,"down_at":23,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6801","nonce":2493296278}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6802","nonce":2493296278}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6804","nonce":2493296278}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6803","nonce":2493296278}]},"public_addr":"172.21.15.83:6801/2493296278","cluster_addr":"172.21.15.83:6802/2493296278","heartbeat_back_addr":"172.21.15.83:6804/2493296278","heartbeat_front_addr":"172.21.15.83:6803/2493296278","state":["exists"]},{"osd":2,"uuid":"8c242dda-3308-4804-b2be-9a1737c42cf6","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":11,"up_thru":15,"down_at":24,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6810","nonce":1553686396}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6812","nonce":1553686396}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6816","nonce":1553686396}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6814","nonce":1553686396}]},"public_addr":"172.21.15.83:6810/1553686396","cluster_addr":"172.21.15.83:6812/1553686396","heartbeat_back_addr":"172.21.15.83:6816/1553686396","heartbeat_front_addr":"172.21.15.83:6814/1553686396","state":["exists"]},{"osd":3,"uuid":"c4dd2c36-3c86-4b56-86fb-fede641a436c","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":11,"up_thru":24,"down_at":25,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6809","nonce":2350261690}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6811","nonce":2350261690}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6815","nonce":2350261690}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6813","nonce":2350261690}]},"public_addr":"172.21.15.83:6809/2350261690","cluster_addr":"172.21.15.83:6811/2350261690","heartbeat_back_addr":"172.21.15.83:6815/2350261690","heartbeat_front_addr":"172.21.15.83:6813/2350261690","state":["exists"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:38:53.047098+0000","dead_epoch":22},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:38:53.354351+0000","dead_epoch":23},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:38:53.422284+0000","dead_epoch":24},{"osd":3,"down_stamp":"2024-09-14T03:40:28.062867+0000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:38:53.620657+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.83:0/293392365":"2024-09-15T03:40:20.028874+0000","172.21.15.83:0/4246762177":"2024-09-15T03:40:20.028874+0000","172.21.15.83:0/3623382676":"2024-09-15T03:40:20.028874+0000","172.21.15.83:6800/1934165865":"2024-09-15T03:40:20.028874+0000","172.21.15.83:0/4208089382":"2024-09-15T03:40:20.028874+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-14T03:40:31.619 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:31.617+0000 7fab0b80f640 1 -- 172.21.15.83:0/955378341 >> v1:172.21.15.83:6800/1803897559 conn(0x7faae80514b0 legacy=0x7faae8053970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:31.619 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:31.617+0000 7fab0b80f640 1 -- 172.21.15.83:0/955378341 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fab0412eff0 msgr2=0x7fab041c5580 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:31.619 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:31.617+0000 7fab0b80f640 1 --2- 172.21.15.83:0/955378341 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fab0412eff0 0x7fab041c5580 secure :-1 s=READY pgs=43 cs=0 l=1 rev1=1 crypto rx=0x7faafc00ec50 tx=0x7faafc00c6a0 comp rx=0 tx=0).stop 2024-09-14T03:40:31.619 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:31.617+0000 7fab0b80f640 1 -- 172.21.15.83:0/955378341 shutdown_connections 2024-09-14T03:40:31.619 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:31.617+0000 7fab0b80f640 1 --2- 172.21.15.83:0/955378341 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fab0412eff0 0x7fab041c5580 unknown :-1 s=CLOSED pgs=43 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:31.619 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:31.617+0000 7fab0b80f640 1 --2- 172.21.15.83:0/955378341 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fab040999a0 0x7fab041c5040 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:31.619 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:31.617+0000 7fab0b80f640 1 --2- 172.21.15.83:0/955378341 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fab04099080 0x7fab041c4b00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:31.619 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:31.617+0000 7fab0b80f640 1 -- 172.21.15.83:0/955378341 >> 172.21.15.83:0/955378341 conn(0x7fab040a1800 msgr2=0x7fab04126f40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:31.620 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:31.617+0000 7fab0b80f640 1 -- 172.21.15.83:0/955378341 shutdown_connections 2024-09-14T03:40:31.620 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:31.617+0000 7fab0b80f640 1 -- 172.21.15.83:0/955378341 wait complete. 2024-09-14T03:40:33.075 INFO:tasks.ceph.osd.3.smithi083.stderr:2024-09-14T03:40:33.073+0000 7f5e6bf20640 -1 osd.3 24 log_to_monitors true 2024-09-14T03:40:33.093 INFO:tasks.ceph.osd.3.smithi083.stderr:2024-09-14T03:40:33.092+0000 7f5e5fccf640 -1 osd.3 24 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-14T03:40:33.125 INFO:tasks.ceph.osd.2.smithi083.stderr:2024-09-14T03:40:33.123+0000 7fd359417640 -1 osd.2 23 log_to_monitors true 2024-09-14T03:40:34.099 INFO:tasks.ceph.osd.2.smithi083.stderr:2024-09-14T03:40:34.097+0000 7fd34cc20640 -1 osd.2 23 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-14T03:40:34.630 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-09-14T03:40:34.743 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:34.741+0000 7f9360685640 1 -- 172.21.15.83:0/1199588389 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f935812c3d0 msgr2=0x7f9358130af0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:34.743 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:34.741+0000 7f9360685640 1 --2- 172.21.15.83:0/1199588389 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f935812c3d0 0x7f9358130af0 secure :-1 s=READY pgs=47 cs=0 l=1 rev1=1 crypto rx=0x7f9348009860 tx=0x7f934802a3d0 comp rx=0 tx=0).stop 2024-09-14T03:40:34.743 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:34.741+0000 7f9360685640 1 -- 172.21.15.83:0/1199588389 shutdown_connections 2024-09-14T03:40:34.743 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:34.741+0000 7f9360685640 1 --2- 172.21.15.83:0/1199588389 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f9358131030 0x7f9358133420 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:34.743 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:34.741+0000 7f9360685640 1 --2- 172.21.15.83:0/1199588389 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f935812c3d0 0x7f9358130af0 unknown :-1 s=CLOSED pgs=47 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:34.743 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:34.741+0000 7f9360685640 1 -- 172.21.15.83:0/1199588389 >> 172.21.15.83:0/1199588389 conn(0x7f935809a2e0 msgr2=0x7f9358097ad0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:34.744 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:34.741+0000 7f9360685640 1 -- 172.21.15.83:0/1199588389 shutdown_connections 2024-09-14T03:40:34.744 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:34.741+0000 7f9360685640 1 -- 172.21.15.83:0/1199588389 wait complete. 2024-09-14T03:40:34.746 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:34.742+0000 7f9360685640 1 Processor -- start 2024-09-14T03:40:34.746 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:34.742+0000 7f9360685640 1 -- start start 2024-09-14T03:40:34.746 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:34.743+0000 7f9360685640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f93580991f0 0x7f935808ce50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:34.746 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:34.743+0000 7f9360685640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f935812c3d0 0x7f935808d390 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:34.746 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:34.743+0000 7f9360685640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f9358131030 0x7f935808d8d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:34.746 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:34.743+0000 7f9360685640 1 -- --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7f9358134670 con 0x7f93580991f0 2024-09-14T03:40:34.746 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:34.743+0000 7f9360685640 1 -- --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7f93581344f0 con 0x7f9358131030 2024-09-14T03:40:34.747 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:34.743+0000 7f9360685640 1 -- --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7f93581347f0 con 0x7f935812c3d0 2024-09-14T03:40:34.747 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:34.743+0000 7f935dc20640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f935812c3d0 0x7f935808d390 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:34.747 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:34.743+0000 7f935e421640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f93580991f0 0x7f935808ce50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:34.747 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:34.743+0000 7f935ec22640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f9358131030 0x7f935808d8d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:34.747 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:34.743+0000 7f935dc20640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f935812c3d0 0x7f935808d390 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3302/0 says I am v2:172.21.15.83:42308/0 (socket says 172.21.15.83:42308) 2024-09-14T03:40:34.747 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:34.743+0000 7f935dc20640 1 -- 172.21.15.83:0/1418531517 learned_addr learned my addr 172.21.15.83:0/1418531517 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:40:34.747 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:34.743+0000 7f935e421640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f93580991f0 0x7f935808ce50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3300/0 says I am v2:172.21.15.83:37140/0 (socket says 172.21.15.83:37140) 2024-09-14T03:40:34.747 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:34.743+0000 7f935ec22640 1 --2- 172.21.15.83:0/1418531517 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f9358131030 0x7f935808d8d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3301/0 says I am v2:172.21.15.83:52894/0 (socket says 172.21.15.83:52894) 2024-09-14T03:40:34.747 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:34.743+0000 7f935e421640 1 -- 172.21.15.83:0/1418531517 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f935812c3d0 msgr2=0x7f935808d390 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:34.747 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:34.743+0000 7f935e421640 1 --2- 172.21.15.83:0/1418531517 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f935812c3d0 0x7f935808d390 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:34.747 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:34.743+0000 7f935e421640 1 -- 172.21.15.83:0/1418531517 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f9358131030 msgr2=0x7f935808d8d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:34.747 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:34.743+0000 7f935e421640 1 --2- 172.21.15.83:0/1418531517 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f9358131030 0x7f935808d8d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:34.747 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:34.743+0000 7f935e421640 1 -- 172.21.15.83:0/1418531517 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f9358090400 con 0x7f93580991f0 2024-09-14T03:40:34.747 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:34.743+0000 7f935ec22640 1 --2- 172.21.15.83:0/1418531517 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f9358131030 0x7f935808d8d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:40:34.748 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:34.743+0000 7f935dc20640 1 --2- 172.21.15.83:0/1418531517 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f935812c3d0 0x7f935808d390 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:40:34.748 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:34.743+0000 7f935e421640 1 --2- 172.21.15.83:0/1418531517 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f93580991f0 0x7f935808ce50 secure :-1 s=READY pgs=63 cs=0 l=1 rev1=1 crypto rx=0x7f935400c3c0 tx=0x7f935400c880 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:40:34.748 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:34.743+0000 7f934effd640 1 -- 172.21.15.83:0/1418531517 <== mon.0 v2:172.21.15.83:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9354011790 con 0x7f93580991f0 2024-09-14T03:40:34.748 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:34.743+0000 7f934effd640 1 -- 172.21.15.83:0/1418531517 <== mon.0 v2:172.21.15.83:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f9354017070 con 0x7f93580991f0 2024-09-14T03:40:34.748 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:34.743+0000 7f9360685640 1 -- 172.21.15.83:0/1418531517 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f935808de10 con 0x7f93580991f0 2024-09-14T03:40:34.748 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:34.743+0000 7f934effd640 1 -- 172.21.15.83:0/1418531517 <== mon.0 v2:172.21.15.83:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9354011ce0 con 0x7f93580991f0 2024-09-14T03:40:34.748 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:34.744+0000 7f9360685640 1 -- 172.21.15.83:0/1418531517 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f93581c8b00 con 0x7f93580991f0 2024-09-14T03:40:34.748 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:34.744+0000 7f9360685640 1 -- 172.21.15.83:0/1418531517 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f935812bcc0 con 0x7f93580991f0 2024-09-14T03:40:34.748 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:34.745+0000 7f934effd640 1 -- 172.21.15.83:0/1418531517 <== mon.0 v2:172.21.15.83:3300/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7f935401c460 con 0x7f93580991f0 2024-09-14T03:40:34.748 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:34.745+0000 7f934effd640 1 -- 172.21.15.83:0/1418531517 <== mon.0 v2:172.21.15.83:3300/0 5 ==== osd_map(29..29 src has 1..29) ==== 3027+0+0 (secure 0 0 0) 0x7f9354068b80 con 0x7f93580991f0 2024-09-14T03:40:34.750 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:34.749+0000 7f934effd640 1 -- 172.21.15.83:0/1418531517 <== mon.0 v2:172.21.15.83:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+157506 (secure 0 0 0) 0x7f9354011e80 con 0x7f93580991f0 2024-09-14T03:40:34.898 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:34.896+0000 7f9360685640 1 -- 172.21.15.83:0/1418531517 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) -- 0x7f93580699c0 con 0x7f93580991f0 2024-09-14T03:40:34.899 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:34.897+0000 7f934effd640 1 -- 172.21.15.83:0/1418531517 <== mon.0 v2:172.21.15.83:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v29) ==== 74+0+9164 (secure 0 0 0) 0x7f9354067c30 con 0x7f93580991f0 2024-09-14T03:40:34.899 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:40:34.899 INFO:teuthology.orchestra.run.smithi083.stdout:{"epoch":29,"fsid":"765298ef-df4c-4f28-af02-b45f60264ff9","created":"2024-09-14T03:38:43.954952+0000","modified":"2024-09-14T03:40:34.086109+0000","last_up_change":"2024-09-14T03:40:34.086109+0000","last_in_change":"2024-09-14T03:38:46.006167+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":8,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":4,"require_min_compat_client":"squid","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-14T03:38:56.922548+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"14","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":4,"score_stable":4,"optimal_score":0.5,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-14T03:39:00.433011+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":18,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":3,"score_stable":3,"optimal_score":0.75,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ff477614-343d-4d97-9df5-94933f9b904d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":9,"last_clean_end":21,"up_from":27,"up_thru":27,"down_at":22,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6801","nonce":3543625573}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6802","nonce":3543625573}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6804","nonce":3543625573}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6803","nonce":3543625573}]},"public_addr":"172.21.15.83:6801/3543625573","cluster_addr":"172.21.15.83:6802/3543625573","heartbeat_back_addr":"172.21.15.83:6804/3543625573","heartbeat_front_addr":"172.21.15.83:6803/3543625573","state":["exists","up"]},{"osd":1,"uuid":"562ec9a0-3f91-4a29-896d-94340e9fa36f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":11,"last_clean_end":21,"up_from":27,"up_thru":27,"down_at":23,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6805","nonce":543231167}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6806","nonce":543231167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6808","nonce":543231167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6807","nonce":543231167}]},"public_addr":"172.21.15.83:6805/543231167","cluster_addr":"172.21.15.83:6806/543231167","heartbeat_back_addr":"172.21.15.83:6808/543231167","heartbeat_front_addr":"172.21.15.83:6807/543231167","state":["exists","up"]},{"osd":2,"uuid":"8c242dda-3308-4804-b2be-9a1737c42cf6","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":11,"up_thru":15,"down_at":24,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6810","nonce":1553686396}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6812","nonce":1553686396}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6816","nonce":1553686396}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6814","nonce":1553686396}]},"public_addr":"172.21.15.83:6810/1553686396","cluster_addr":"172.21.15.83:6812/1553686396","heartbeat_back_addr":"172.21.15.83:6816/1553686396","heartbeat_front_addr":"172.21.15.83:6814/1553686396","state":["exists"]},{"osd":3,"uuid":"c4dd2c36-3c86-4b56-86fb-fede641a436c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":11,"last_clean_end":24,"up_from":29,"up_thru":24,"down_at":25,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6813","nonce":3860913217}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6814","nonce":3860913217}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6816","nonce":3860913217}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6815","nonce":3860913217}]},"public_addr":"172.21.15.83:6813/3860913217","cluster_addr":"172.21.15.83:6814/3860913217","heartbeat_back_addr":"172.21.15.83:6816/3860913217","heartbeat_front_addr":"172.21.15.83:6815/3860913217","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:38:53.047098+0000","dead_epoch":22},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:38:53.354351+0000","dead_epoch":23},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:38:53.422284+0000","dead_epoch":24},{"osd":3,"down_stamp":"2024-09-14T03:40:28.062867+0000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:38:53.620657+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.83:0/293392365":"2024-09-15T03:40:20.028874+0000","172.21.15.83:0/4246762177":"2024-09-15T03:40:20.028874+0000","172.21.15.83:0/3623382676":"2024-09-15T03:40:20.028874+0000","172.21.15.83:6800/1934165865":"2024-09-15T03:40:20.028874+0000","172.21.15.83:0/4208089382":"2024-09-15T03:40:20.028874+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-14T03:40:34.901 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:34.899+0000 7f9360685640 1 -- 172.21.15.83:0/1418531517 >> v1:172.21.15.83:6800/1803897559 conn(0x7f933c0514e0 legacy=0x7f933c0539a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:34.901 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:34.899+0000 7f9360685640 1 -- 172.21.15.83:0/1418531517 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f93580991f0 msgr2=0x7f935808ce50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:34.901 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:34.899+0000 7f9360685640 1 --2- 172.21.15.83:0/1418531517 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f93580991f0 0x7f935808ce50 secure :-1 s=READY pgs=63 cs=0 l=1 rev1=1 crypto rx=0x7f935400c3c0 tx=0x7f935400c880 comp rx=0 tx=0).stop 2024-09-14T03:40:34.901 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:34.899+0000 7f9360685640 1 -- 172.21.15.83:0/1418531517 shutdown_connections 2024-09-14T03:40:34.901 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:34.899+0000 7f9360685640 1 --2- 172.21.15.83:0/1418531517 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f9358131030 0x7f935808d8d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:34.902 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:34.899+0000 7f9360685640 1 --2- 172.21.15.83:0/1418531517 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f935812c3d0 0x7f935808d390 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:34.902 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:34.899+0000 7f9360685640 1 --2- 172.21.15.83:0/1418531517 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f93580991f0 0x7f935808ce50 unknown :-1 s=CLOSED pgs=63 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:34.902 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:34.899+0000 7f9360685640 1 -- 172.21.15.83:0/1418531517 >> 172.21.15.83:0/1418531517 conn(0x7f935809a2e0 msgr2=0x7f935809aed0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:34.902 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:34.899+0000 7f9360685640 1 -- 172.21.15.83:0/1418531517 shutdown_connections 2024-09-14T03:40:34.903 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:34.899+0000 7f9360685640 1 -- 172.21.15.83:0/1418531517 wait complete. 2024-09-14T03:40:37.912 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-09-14T03:40:38.026 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.024+0000 7ffa75273640 1 -- 172.21.15.83:0/425010119 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7ffa700999a0 msgr2=0x7ffa7012eab0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:38.026 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.024+0000 7ffa75273640 1 --2- 172.21.15.83:0/425010119 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7ffa700999a0 0x7ffa7012eab0 secure :-1 s=READY pgs=46 cs=0 l=1 rev1=1 crypto rx=0x7ffa58009900 tx=0x7ffa5802a530 comp rx=0 tx=0).stop 2024-09-14T03:40:38.026 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.024+0000 7ffa75273640 1 -- 172.21.15.83:0/425010119 shutdown_connections 2024-09-14T03:40:38.026 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.024+0000 7ffa75273640 1 --2- 172.21.15.83:0/425010119 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7ffa700999a0 0x7ffa7012eab0 unknown :-1 s=CLOSED pgs=46 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:38.026 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.024+0000 7ffa75273640 1 --2- 172.21.15.83:0/425010119 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7ffa70099080 0x7ffa70099460 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:38.026 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.024+0000 7ffa75273640 1 -- 172.21.15.83:0/425010119 >> 172.21.15.83:0/425010119 conn(0x7ffa700a1800 msgr2=0x7ffa700a1c10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:38.026 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.024+0000 7ffa75273640 1 -- 172.21.15.83:0/425010119 shutdown_connections 2024-09-14T03:40:38.026 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.024+0000 7ffa75273640 1 -- 172.21.15.83:0/425010119 wait complete. 2024-09-14T03:40:38.029 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.025+0000 7ffa75273640 1 Processor -- start 2024-09-14T03:40:38.029 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.025+0000 7ffa75273640 1 -- start start 2024-09-14T03:40:38.029 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.026+0000 7ffa75273640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7ffa70099080 0x7ffa70090390 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:38.029 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.026+0000 7ffa75273640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7ffa700999a0 0x7ffa700908d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:38.029 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.026+0000 7ffa75273640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7ffa7012eff0 0x7ffa70090e10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:38.029 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.026+0000 7ffa75273640 1 -- --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7ffa701376d0 con 0x7ffa70099080 2024-09-14T03:40:38.029 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.026+0000 7ffa75273640 1 -- --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7ffa70137550 con 0x7ffa7012eff0 2024-09-14T03:40:38.029 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.026+0000 7ffa75273640 1 -- --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7ffa70137850 con 0x7ffa700999a0 2024-09-14T03:40:38.029 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.026+0000 7ffa6effd640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7ffa70099080 0x7ffa70090390 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:38.029 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.026+0000 7ffa6e7fc640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7ffa700999a0 0x7ffa700908d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:38.029 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.026+0000 7ffa6f7fe640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7ffa7012eff0 0x7ffa70090e10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:38.029 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.026+0000 7ffa6e7fc640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7ffa700999a0 0x7ffa700908d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3302/0 says I am v2:172.21.15.83:42318/0 (socket says 172.21.15.83:42318) 2024-09-14T03:40:38.029 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.026+0000 7ffa6e7fc640 1 -- 172.21.15.83:0/1318355591 learned_addr learned my addr 172.21.15.83:0/1318355591 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:40:38.030 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.026+0000 7ffa6effd640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7ffa70099080 0x7ffa70090390 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3300/0 says I am v2:172.21.15.83:37150/0 (socket says 172.21.15.83:37150) 2024-09-14T03:40:38.030 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.026+0000 7ffa6e7fc640 1 -- 172.21.15.83:0/1318355591 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7ffa7012eff0 msgr2=0x7ffa70090e10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:38.030 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.026+0000 7ffa6e7fc640 1 --2- 172.21.15.83:0/1318355591 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7ffa7012eff0 0x7ffa70090e10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:38.030 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.026+0000 7ffa6e7fc640 1 -- 172.21.15.83:0/1318355591 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7ffa70099080 msgr2=0x7ffa70090390 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:38.030 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.026+0000 7ffa6e7fc640 1 --2- 172.21.15.83:0/1318355591 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7ffa70099080 0x7ffa70090390 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:38.030 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.026+0000 7ffa6e7fc640 1 -- 172.21.15.83:0/1318355591 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7ffa700918c0 con 0x7ffa700999a0 2024-09-14T03:40:38.030 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.026+0000 7ffa6f7fe640 1 --2- 172.21.15.83:0/1318355591 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7ffa7012eff0 0x7ffa70090e10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:40:38.030 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.026+0000 7ffa6effd640 1 --2- 172.21.15.83:0/1318355591 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7ffa70099080 0x7ffa70090390 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:40:38.030 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.026+0000 7ffa6e7fc640 1 --2- 172.21.15.83:0/1318355591 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7ffa700999a0 0x7ffa700908d0 secure :-1 s=READY pgs=48 cs=0 l=1 rev1=1 crypto rx=0x7ffa58000c00 tx=0x7ffa580038a0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:40:38.030 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.026+0000 7ffa677fe640 1 -- 172.21.15.83:0/1318355591 <== mon.2 v2:172.21.15.83:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ffa5803f070 con 0x7ffa700999a0 2024-09-14T03:40:38.030 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.026+0000 7ffa677fe640 1 -- 172.21.15.83:0/1318355591 <== mon.2 v2:172.21.15.83:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7ffa580047a0 con 0x7ffa700999a0 2024-09-14T03:40:38.030 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.026+0000 7ffa75273640 1 -- 172.21.15.83:0/1318355591 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7ffa70098190 con 0x7ffa700999a0 2024-09-14T03:40:38.030 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.026+0000 7ffa677fe640 1 -- 172.21.15.83:0/1318355591 <== mon.2 v2:172.21.15.83:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ffa58035030 con 0x7ffa700999a0 2024-09-14T03:40:38.030 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.026+0000 7ffa75273640 1 -- 172.21.15.83:0/1318355591 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7ffa700941e0 con 0x7ffa700999a0 2024-09-14T03:40:38.031 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.028+0000 7ffa75273640 1 -- 172.21.15.83:0/1318355591 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7ffa70099460 con 0x7ffa700999a0 2024-09-14T03:40:38.031 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.028+0000 7ffa677fe640 1 -- 172.21.15.83:0/1318355591 <== mon.2 v2:172.21.15.83:3302/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7ffa58042070 con 0x7ffa700999a0 2024-09-14T03:40:38.031 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.028+0000 7ffa677fe640 1 -- 172.21.15.83:0/1318355591 <== mon.2 v2:172.21.15.83:3302/0 5 ==== osd_map(31..31 src has 1..31) ==== 3027+0+0 (secure 0 0 0) 0x7ffa58089d40 con 0x7ffa700999a0 2024-09-14T03:40:38.033 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.032+0000 7ffa677fe640 1 -- 172.21.15.83:0/1318355591 <== mon.2 v2:172.21.15.83:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+157506 (secure 0 0 0) 0x7ffa700918c0 con 0x7ffa700999a0 2024-09-14T03:40:38.184 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.183+0000 7ffa75273640 1 -- 172.21.15.83:0/1318355591 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) -- 0x7ffa700698c0 con 0x7ffa700999a0 2024-09-14T03:40:38.185 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.183+0000 7ffa677fe640 1 -- 172.21.15.83:0/1318355591 <== mon.2 v2:172.21.15.83:3302/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v31) ==== 74+0+9162 (secure 0 0 0) 0x7ffa58088df0 con 0x7ffa700999a0 2024-09-14T03:40:38.185 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:40:38.185 INFO:teuthology.orchestra.run.smithi083.stdout:{"epoch":31,"fsid":"765298ef-df4c-4f28-af02-b45f60264ff9","created":"2024-09-14T03:38:43.954952+0000","modified":"2024-09-14T03:40:36.101335+0000","last_up_change":"2024-09-14T03:40:35.100253+0000","last_in_change":"2024-09-14T03:38:46.006167+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":8,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":4,"require_min_compat_client":"squid","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-14T03:38:56.922548+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"14","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":4,"score_stable":4,"optimal_score":0.5,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-14T03:39:00.433011+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":18,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.5,"score_stable":2.5,"optimal_score":1,"raw_score_acting":2.5,"raw_score_stable":2.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ff477614-343d-4d97-9df5-94933f9b904d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":9,"last_clean_end":21,"up_from":27,"up_thru":27,"down_at":22,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6801","nonce":3543625573}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6802","nonce":3543625573}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6804","nonce":3543625573}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6803","nonce":3543625573}]},"public_addr":"172.21.15.83:6801/3543625573","cluster_addr":"172.21.15.83:6802/3543625573","heartbeat_back_addr":"172.21.15.83:6804/3543625573","heartbeat_front_addr":"172.21.15.83:6803/3543625573","state":["exists","up"]},{"osd":1,"uuid":"562ec9a0-3f91-4a29-896d-94340e9fa36f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":11,"last_clean_end":21,"up_from":27,"up_thru":27,"down_at":23,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6805","nonce":543231167}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6806","nonce":543231167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6808","nonce":543231167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6807","nonce":543231167}]},"public_addr":"172.21.15.83:6805/543231167","cluster_addr":"172.21.15.83:6806/543231167","heartbeat_back_addr":"172.21.15.83:6808/543231167","heartbeat_front_addr":"172.21.15.83:6807/543231167","state":["exists","up"]},{"osd":2,"uuid":"8c242dda-3308-4804-b2be-9a1737c42cf6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":11,"last_clean_end":23,"up_from":30,"up_thru":30,"down_at":24,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6809","nonce":905248239}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6810","nonce":905248239}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6812","nonce":905248239}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6811","nonce":905248239}]},"public_addr":"172.21.15.83:6809/905248239","cluster_addr":"172.21.15.83:6810/905248239","heartbeat_back_addr":"172.21.15.83:6812/905248239","heartbeat_front_addr":"172.21.15.83:6811/905248239","state":["exists","up"]},{"osd":3,"uuid":"c4dd2c36-3c86-4b56-86fb-fede641a436c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":11,"last_clean_end":24,"up_from":29,"up_thru":30,"down_at":25,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6813","nonce":3860913217}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6814","nonce":3860913217}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6816","nonce":3860913217}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6815","nonce":3860913217}]},"public_addr":"172.21.15.83:6813/3860913217","cluster_addr":"172.21.15.83:6814/3860913217","heartbeat_back_addr":"172.21.15.83:6816/3860913217","heartbeat_front_addr":"172.21.15.83:6815/3860913217","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:38:53.047098+0000","dead_epoch":22},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:38:53.354351+0000","dead_epoch":23},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:38:53.422284+0000","dead_epoch":24},{"osd":3,"down_stamp":"2024-09-14T03:40:28.062867+0000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:38:53.620657+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.83:0/293392365":"2024-09-15T03:40:20.028874+0000","172.21.15.83:0/4246762177":"2024-09-15T03:40:20.028874+0000","172.21.15.83:0/3623382676":"2024-09-15T03:40:20.028874+0000","172.21.15.83:6800/1934165865":"2024-09-15T03:40:20.028874+0000","172.21.15.83:0/4208089382":"2024-09-15T03:40:20.028874+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-14T03:40:38.187 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.185+0000 7ffa75273640 1 -- 172.21.15.83:0/1318355591 >> v1:172.21.15.83:6800/1803897559 conn(0x7ffa500514e0 legacy=0x7ffa500539a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:38.187 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.185+0000 7ffa75273640 1 -- 172.21.15.83:0/1318355591 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7ffa700999a0 msgr2=0x7ffa700908d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:38.187 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.185+0000 7ffa75273640 1 --2- 172.21.15.83:0/1318355591 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7ffa700999a0 0x7ffa700908d0 secure :-1 s=READY pgs=48 cs=0 l=1 rev1=1 crypto rx=0x7ffa58000c00 tx=0x7ffa580038a0 comp rx=0 tx=0).stop 2024-09-14T03:40:38.187 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.185+0000 7ffa75273640 1 -- 172.21.15.83:0/1318355591 shutdown_connections 2024-09-14T03:40:38.187 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.185+0000 7ffa75273640 1 --2- 172.21.15.83:0/1318355591 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7ffa7012eff0 0x7ffa70090e10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:38.187 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.185+0000 7ffa75273640 1 --2- 172.21.15.83:0/1318355591 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7ffa700999a0 0x7ffa700908d0 unknown :-1 s=CLOSED pgs=48 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:38.188 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.185+0000 7ffa75273640 1 --2- 172.21.15.83:0/1318355591 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7ffa70099080 0x7ffa70090390 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:38.188 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.185+0000 7ffa75273640 1 -- 172.21.15.83:0/1318355591 >> 172.21.15.83:0/1318355591 conn(0x7ffa700a1800 msgr2=0x7ffa7012b6f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:38.188 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.185+0000 7ffa75273640 1 -- 172.21.15.83:0/1318355591 shutdown_connections 2024-09-14T03:40:38.188 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.185+0000 7ffa75273640 1 -- 172.21.15.83:0/1318355591 wait complete. 2024-09-14T03:40:38.198 INFO:tasks.ceph.ceph_manager.ceph:all up! 2024-09-14T03:40:38.198 INFO:teuthology.run_tasks:Running task exec... 2024-09-14T03:40:38.208 INFO:teuthology.task.exec:Executing custom commands... 2024-09-14T03:40:38.208 INFO:teuthology.task.exec:Running commands on role mon.a host ubuntu@smithi083.front.sepia.ceph.com 2024-09-14T03:40:38.208 DEBUG:teuthology.orchestra.run.smithi083:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'ceph versions' 2024-09-14T03:40:38.305 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.303+0000 7f5e4f59e640 1 -- 172.21.15.83:0/2952217677 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f5e5006f420 msgr2=0x7f5e5006f800 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:38.305 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.303+0000 7f5e4f59e640 1 --2- 172.21.15.83:0/2952217677 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f5e5006f420 0x7f5e5006f800 secure :-1 s=READY pgs=66 cs=0 l=1 rev1=1 crypto rx=0x7f5e48009830 tx=0x7f5e4802a3f0 comp rx=0 tx=0).stop 2024-09-14T03:40:38.305 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.303+0000 7f5e4f59e640 1 -- 172.21.15.83:0/2952217677 shutdown_connections 2024-09-14T03:40:38.305 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.303+0000 7f5e4f59e640 1 --2- 172.21.15.83:0/2952217677 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f5e5012b620 0x7f5e5012bab0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:38.305 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.303+0000 7f5e4f59e640 1 --2- 172.21.15.83:0/2952217677 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f5e5006f420 0x7f5e5006f800 unknown :-1 s=CLOSED pgs=66 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:38.305 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.303+0000 7f5e4f59e640 1 -- 172.21.15.83:0/2952217677 >> 172.21.15.83:0/2952217677 conn(0x7f5e5009dae0 msgr2=0x7f5e5009def0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:38.306 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.303+0000 7f5e4f59e640 1 -- 172.21.15.83:0/2952217677 shutdown_connections 2024-09-14T03:40:38.306 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.303+0000 7f5e4f59e640 1 -- 172.21.15.83:0/2952217677 wait complete. 2024-09-14T03:40:38.308 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.304+0000 7f5e4f59e640 1 Processor -- start 2024-09-14T03:40:38.308 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.304+0000 7f5e4f59e640 1 -- start start 2024-09-14T03:40:38.308 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.304+0000 7f5e4f59e640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f5e5006f420 0x7f5e50126f60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:38.308 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.304+0000 7f5e4f59e640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f5e5006fd40 0x7f5e501274a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:38.308 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.304+0000 7f5e4f59e640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f5e5012b620 0x7f5e501c2480 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:38.308 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.305+0000 7f5e4f59e640 1 -- --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7f5e501335e0 con 0x7f5e5006fd40 2024-09-14T03:40:38.308 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.305+0000 7f5e4f59e640 1 -- --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7f5e50133460 con 0x7f5e5006f420 2024-09-14T03:40:38.308 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.305+0000 7f5e4f59e640 1 -- --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7f5e50133760 con 0x7f5e5012b620 2024-09-14T03:40:38.308 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.305+0000 7f5e4e59c640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f5e5006f420 0x7f5e50126f60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:38.308 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.305+0000 7f5e4ed9d640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f5e5012b620 0x7f5e501c2480 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:38.309 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.305+0000 7f5e4dd9b640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f5e5006fd40 0x7f5e501274a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:38.309 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.305+0000 7f5e4e59c640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f5e5006f420 0x7f5e50126f60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3301/0 says I am v2:172.21.15.83:52928/0 (socket says 172.21.15.83:52928) 2024-09-14T03:40:38.309 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.305+0000 7f5e4e59c640 1 -- 172.21.15.83:0/3190786188 learned_addr learned my addr 172.21.15.83:0/3190786188 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:40:38.309 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.305+0000 7f5e4dd9b640 1 --2- 172.21.15.83:0/3190786188 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f5e5006fd40 0x7f5e501274a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3300/0 says I am v2:172.21.15.83:37178/0 (socket says 172.21.15.83:37178) 2024-09-14T03:40:38.309 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.305+0000 7f5e4ed9d640 1 --2- 172.21.15.83:0/3190786188 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f5e5012b620 0x7f5e501c2480 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3302/0 says I am v2:172.21.15.83:42336/0 (socket says 172.21.15.83:42336) 2024-09-14T03:40:38.309 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.305+0000 7f5e4dd9b640 1 -- 172.21.15.83:0/3190786188 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f5e5012b620 msgr2=0x7f5e501c2480 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:38.309 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.305+0000 7f5e4dd9b640 1 --2- 172.21.15.83:0/3190786188 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f5e5012b620 0x7f5e501c2480 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:38.309 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.305+0000 7f5e4dd9b640 1 -- 172.21.15.83:0/3190786188 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f5e5006f420 msgr2=0x7f5e50126f60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:38.309 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.305+0000 7f5e4dd9b640 1 --2- 172.21.15.83:0/3190786188 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f5e5006f420 0x7f5e50126f60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:38.309 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.305+0000 7f5e4dd9b640 1 -- 172.21.15.83:0/3190786188 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f5e5012cf90 con 0x7f5e5006fd40 2024-09-14T03:40:38.309 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.305+0000 7f5e4ed9d640 1 --2- 172.21.15.83:0/3190786188 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f5e5012b620 0x7f5e501c2480 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:40:38.309 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.305+0000 7f5e4e59c640 1 --2- 172.21.15.83:0/3190786188 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f5e5006f420 0x7f5e50126f60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:40:38.309 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.305+0000 7f5e4dd9b640 1 --2- 172.21.15.83:0/3190786188 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f5e5006fd40 0x7f5e501274a0 secure :-1 s=READY pgs=67 cs=0 l=1 rev1=1 crypto rx=0x7f5e3800cb40 tx=0x7f5e3800cf40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:40:38.309 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.305+0000 7f5e46ffd640 1 -- 172.21.15.83:0/3190786188 <== mon.0 v2:172.21.15.83:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5e38016020 con 0x7f5e5006fd40 2024-09-14T03:40:38.310 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.305+0000 7f5e46ffd640 1 -- 172.21.15.83:0/3190786188 <== mon.0 v2:172.21.15.83:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f5e3800ad20 con 0x7f5e5006fd40 2024-09-14T03:40:38.310 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.305+0000 7f5e46ffd640 1 -- 172.21.15.83:0/3190786188 <== mon.0 v2:172.21.15.83:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5e38012680 con 0x7f5e5006fd40 2024-09-14T03:40:38.310 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.305+0000 7f5e4f59e640 1 -- 172.21.15.83:0/3190786188 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f5e5012ca90 con 0x7f5e5006fd40 2024-09-14T03:40:38.310 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.305+0000 7f5e4f59e640 1 -- 172.21.15.83:0/3190786188 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f5e501279e0 con 0x7f5e5006fd40 2024-09-14T03:40:38.310 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.306+0000 7f5e4f59e640 1 -- 172.21.15.83:0/3190786188 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f5e5006f800 con 0x7f5e5006fd40 2024-09-14T03:40:38.310 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.306+0000 7f5e46ffd640 1 -- 172.21.15.83:0/3190786188 <== mon.0 v2:172.21.15.83:3300/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7f5e38019020 con 0x7f5e5006fd40 2024-09-14T03:40:38.310 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.307+0000 7f5e46ffd640 1 -- 172.21.15.83:0/3190786188 <== mon.0 v2:172.21.15.83:3300/0 5 ==== osd_map(31..31 src has 1..31) ==== 3027+0+0 (secure 0 0 0) 0x7f5e38069170 con 0x7f5e5006fd40 2024-09-14T03:40:38.312 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.310+0000 7f5e46ffd640 1 -- 172.21.15.83:0/3190786188 <== mon.0 v2:172.21.15.83:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+157506 (secure 0 0 0) 0x7f5e38020030 con 0x7f5e5006fd40 2024-09-14T03:40:38.515 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.514+0000 7f5e4f59e640 1 -- 172.21.15.83:0/3190786188 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_command({"prefix": "versions"} v 0) -- 0x7f5e500689e0 con 0x7f5e5006fd40 2024-09-14T03:40:38.515 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.514+0000 7f5e46ffd640 1 -- 172.21.15.83:0/3190786188 <== mon.0 v2:172.21.15.83:3300/0 7 ==== mon_command_ack([{"prefix": "versions"}]=0 v0) ==== 56+0+499 (secure 0 0 0) 0x7f5e38068220 con 0x7f5e5006fd40 2024-09-14T03:40:38.516 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-09-14T03:40:38.516 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2024-09-14T03:40:38.516 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 3 2024-09-14T03:40:38.516 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-14T03:40:38.516 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2024-09-14T03:40:38.516 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 2 2024-09-14T03:40:38.516 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-14T03:40:38.516 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2024-09-14T03:40:38.516 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 4 2024-09-14T03:40:38.516 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-14T03:40:38.517 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2024-09-14T03:40:38.517 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 9 2024-09-14T03:40:38.517 INFO:teuthology.orchestra.run.smithi083.stdout: } 2024-09-14T03:40:38.517 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-09-14T03:40:38.517 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.516+0000 7f5e4f59e640 1 -- 172.21.15.83:0/3190786188 >> v1:172.21.15.83:6800/1803897559 conn(0x7f5e300514e0 legacy=0x7f5e300539a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:38.517 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.516+0000 7f5e4f59e640 1 -- 172.21.15.83:0/3190786188 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f5e5006fd40 msgr2=0x7f5e501274a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:38.518 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.516+0000 7f5e4f59e640 1 --2- 172.21.15.83:0/3190786188 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f5e5006fd40 0x7f5e501274a0 secure :-1 s=READY pgs=67 cs=0 l=1 rev1=1 crypto rx=0x7f5e3800cb40 tx=0x7f5e3800cf40 comp rx=0 tx=0).stop 2024-09-14T03:40:38.518 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.516+0000 7f5e4f59e640 1 -- 172.21.15.83:0/3190786188 shutdown_connections 2024-09-14T03:40:38.518 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.516+0000 7f5e4f59e640 1 --2- 172.21.15.83:0/3190786188 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f5e5012b620 0x7f5e501c2480 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:38.518 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.516+0000 7f5e4f59e640 1 --2- 172.21.15.83:0/3190786188 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f5e5006fd40 0x7f5e501274a0 unknown :-1 s=CLOSED pgs=67 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:38.518 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.516+0000 7f5e4f59e640 1 --2- 172.21.15.83:0/3190786188 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f5e5006f420 0x7f5e50126f60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:38.518 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.516+0000 7f5e4f59e640 1 -- 172.21.15.83:0/3190786188 >> 172.21.15.83:0/3190786188 conn(0x7f5e5009dae0 msgr2=0x7f5e50095160 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:38.518 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.516+0000 7f5e4f59e640 1 -- 172.21.15.83:0/3190786188 shutdown_connections 2024-09-14T03:40:38.518 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.516+0000 7f5e4f59e640 1 -- 172.21.15.83:0/3190786188 wait complete. 2024-09-14T03:40:38.528 DEBUG:teuthology.orchestra.run.smithi083:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'ceph osd dump -f json-pretty' 2024-09-14T03:40:38.625 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.623+0000 7fb2b6227640 1 -- 172.21.15.83:0/3274943507 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fb2b00924c0 msgr2=0x7fb2b0092930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:38.625 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.623+0000 7fb2b6227640 1 --2- 172.21.15.83:0/3274943507 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fb2b00924c0 0x7fb2b0092930 secure :-1 s=READY pgs=68 cs=0 l=1 rev1=1 crypto rx=0x7fb2a8005fd0 tx=0x7fb2a802a770 comp rx=0 tx=0).stop 2024-09-14T03:40:38.625 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.623+0000 7fb2b6227640 1 -- 172.21.15.83:0/3274943507 shutdown_connections 2024-09-14T03:40:38.625 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.623+0000 7fb2b6227640 1 --2- 172.21.15.83:0/3274943507 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fb2b00924c0 0x7fb2b0092930 unknown :-1 s=CLOSED pgs=68 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:38.625 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.623+0000 7fb2b6227640 1 --2- 172.21.15.83:0/3274943507 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fb2b012c180 0x7fb2b0091f80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:38.625 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.623+0000 7fb2b6227640 1 --2- 172.21.15.83:0/3274943507 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fb2b012b860 0x7fb2b012bc40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:38.625 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.623+0000 7fb2b6227640 1 -- 172.21.15.83:0/3274943507 >> 172.21.15.83:0/3274943507 conn(0x7fb2b009b580 msgr2=0x7fb2b009b990 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:38.625 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.623+0000 7fb2b6227640 1 -- 172.21.15.83:0/3274943507 shutdown_connections 2024-09-14T03:40:38.625 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.623+0000 7fb2b6227640 1 -- 172.21.15.83:0/3274943507 wait complete. 2024-09-14T03:40:38.628 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.624+0000 7fb2b6227640 1 Processor -- start 2024-09-14T03:40:38.628 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.624+0000 7fb2b6227640 1 -- start start 2024-09-14T03:40:38.628 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.624+0000 7fb2b6227640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fb2b00924c0 0x7fb2b01c4dd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:38.628 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.624+0000 7fb2b6227640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fb2b012b860 0x7fb2b01c5310 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:38.628 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.624+0000 7fb2b6227640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fb2b012c180 0x7fb2b0072dc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:38.628 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.624+0000 7fb2b6227640 1 -- --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7fb2b0091de0 con 0x7fb2b012b860 2024-09-14T03:40:38.628 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.624+0000 7fb2b6227640 1 -- --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7fb2b0091c60 con 0x7fb2b00924c0 2024-09-14T03:40:38.628 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.624+0000 7fb2b6227640 1 -- --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7fb2b0132840 con 0x7fb2b012c180 2024-09-14T03:40:38.628 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.624+0000 7fb2af7fe640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fb2b00924c0 0x7fb2b01c4dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:38.628 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.624+0000 7fb2aeffd640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fb2b012b860 0x7fb2b01c5310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:38.629 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.625+0000 7fb2aeffd640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fb2b012b860 0x7fb2b01c5310 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3300/0 says I am v2:172.21.15.83:37194/0 (socket says 172.21.15.83:37194) 2024-09-14T03:40:38.629 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.625+0000 7fb2aeffd640 1 -- 172.21.15.83:0/3672203891 learned_addr learned my addr 172.21.15.83:0/3672203891 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:40:38.629 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.625+0000 7fb2affff640 1 --2- 172.21.15.83:0/3672203891 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fb2b012c180 0x7fb2b0072dc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:38.629 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.625+0000 7fb2af7fe640 1 -- 172.21.15.83:0/3672203891 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fb2b012c180 msgr2=0x7fb2b0072dc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:38.629 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.625+0000 7fb2af7fe640 1 --2- 172.21.15.83:0/3672203891 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fb2b012c180 0x7fb2b0072dc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:38.629 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.625+0000 7fb2af7fe640 1 -- 172.21.15.83:0/3672203891 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fb2b012b860 msgr2=0x7fb2b01c5310 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:38.629 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.625+0000 7fb2af7fe640 1 --2- 172.21.15.83:0/3672203891 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fb2b012b860 0x7fb2b01c5310 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:38.629 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.625+0000 7fb2af7fe640 1 -- 172.21.15.83:0/3672203891 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fb2b008d0e0 con 0x7fb2b00924c0 2024-09-14T03:40:38.629 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.625+0000 7fb2aeffd640 1 --2- 172.21.15.83:0/3672203891 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fb2b012b860 0x7fb2b01c5310 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:40:38.629 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.625+0000 7fb2affff640 1 --2- 172.21.15.83:0/3672203891 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fb2b012c180 0x7fb2b0072dc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:40:38.629 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.625+0000 7fb2af7fe640 1 --2- 172.21.15.83:0/3672203891 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fb2b00924c0 0x7fb2b01c4dd0 secure :-1 s=READY pgs=47 cs=0 l=1 rev1=1 crypto rx=0x7fb2a000ea60 tx=0x7fb2a000ef20 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:40:38.629 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.625+0000 7fb297fff640 1 -- 172.21.15.83:0/3672203891 <== mon.1 v2:172.21.15.83:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb2a000cd40 con 0x7fb2b00924c0 2024-09-14T03:40:38.629 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.625+0000 7fb297fff640 1 -- 172.21.15.83:0/3672203891 <== mon.1 v2:172.21.15.83:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fb2a0004020 con 0x7fb2b00924c0 2024-09-14T03:40:38.629 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.625+0000 7fb2b6227640 1 -- 172.21.15.83:0/3672203891 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fb2b008d520 con 0x7fb2b00924c0 2024-09-14T03:40:38.630 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.625+0000 7fb297fff640 1 -- 172.21.15.83:0/3672203891 <== mon.1 v2:172.21.15.83:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb2a0004430 con 0x7fb2b00924c0 2024-09-14T03:40:38.630 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.625+0000 7fb2b6227640 1 -- 172.21.15.83:0/3672203891 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7fb2b01c5850 con 0x7fb2b00924c0 2024-09-14T03:40:38.630 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.626+0000 7fb2b6227640 1 -- 172.21.15.83:0/3672203891 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fb2b0092930 con 0x7fb2b00924c0 2024-09-14T03:40:38.630 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.626+0000 7fb297fff640 1 -- 172.21.15.83:0/3672203891 <== mon.1 v2:172.21.15.83:3301/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7fb2a0005020 con 0x7fb2b00924c0 2024-09-14T03:40:38.630 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.627+0000 7fb297fff640 1 -- 172.21.15.83:0/3672203891 <== mon.1 v2:172.21.15.83:3301/0 5 ==== osd_map(31..31 src has 1..31) ==== 3027+0+0 (secure 0 0 0) 0x7fb2a0069380 con 0x7fb2b00924c0 2024-09-14T03:40:38.631 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.630+0000 7fb297fff640 1 -- 172.21.15.83:0/3672203891 <== mon.1 v2:172.21.15.83:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+157506 (secure 0 0 0) 0x7fb2a0068430 con 0x7fb2b00924c0 2024-09-14T03:40:38.780 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.779+0000 7fb2b6227640 1 -- 172.21.15.83:0/3672203891 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_command({"prefix": "osd dump", "format": "json-pretty"} v 0) -- 0x7fb2b0074a10 con 0x7fb2b00924c0 2024-09-14T03:40:38.781 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.779+0000 7fb297fff640 1 -- 172.21.15.83:0/3672203891 <== mon.1 v2:172.21.15.83:3301/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json-pretty"}]=0 v31) ==== 81+0+16826 (secure 0 0 0) 0x7fb2a0067f70 con 0x7fb2b00924c0 2024-09-14T03:40:38.781 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:40:38.781 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-09-14T03:40:38.781 INFO:teuthology.orchestra.run.smithi083.stdout: "epoch": 31, 2024-09-14T03:40:38.782 INFO:teuthology.orchestra.run.smithi083.stdout: "fsid": "765298ef-df4c-4f28-af02-b45f60264ff9", 2024-09-14T03:40:38.782 INFO:teuthology.orchestra.run.smithi083.stdout: "created": "2024-09-14T03:38:43.954952+0000", 2024-09-14T03:40:38.782 INFO:teuthology.orchestra.run.smithi083.stdout: "modified": "2024-09-14T03:40:36.101335+0000", 2024-09-14T03:40:38.782 INFO:teuthology.orchestra.run.smithi083.stdout: "last_up_change": "2024-09-14T03:40:35.100253+0000", 2024-09-14T03:40:38.782 INFO:teuthology.orchestra.run.smithi083.stdout: "last_in_change": "2024-09-14T03:38:46.006167+0000", 2024-09-14T03:40:38.782 INFO:teuthology.orchestra.run.smithi083.stdout: "flags": "sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit", 2024-09-14T03:40:38.782 INFO:teuthology.orchestra.run.smithi083.stdout: "flags_num": 5799936, 2024-09-14T03:40:38.782 INFO:teuthology.orchestra.run.smithi083.stdout: "flags_set": [ 2024-09-14T03:40:38.782 INFO:teuthology.orchestra.run.smithi083.stdout: "pglog_hardlimit", 2024-09-14T03:40:38.782 INFO:teuthology.orchestra.run.smithi083.stdout: "purged_snapdirs", 2024-09-14T03:40:38.782 INFO:teuthology.orchestra.run.smithi083.stdout: "recovery_deletes", 2024-09-14T03:40:38.782 INFO:teuthology.orchestra.run.smithi083.stdout: "sortbitwise" 2024-09-14T03:40:38.782 INFO:teuthology.orchestra.run.smithi083.stdout: ], 2024-09-14T03:40:38.782 INFO:teuthology.orchestra.run.smithi083.stdout: "crush_version": 8, 2024-09-14T03:40:38.783 INFO:teuthology.orchestra.run.smithi083.stdout: "full_ratio": 0.94999998807907104, 2024-09-14T03:40:38.783 INFO:teuthology.orchestra.run.smithi083.stdout: "backfillfull_ratio": 0.89999997615814209, 2024-09-14T03:40:38.783 INFO:teuthology.orchestra.run.smithi083.stdout: "nearfull_ratio": 0.85000002384185791, 2024-09-14T03:40:38.783 INFO:teuthology.orchestra.run.smithi083.stdout: "cluster_snapshot": "", 2024-09-14T03:40:38.783 INFO:teuthology.orchestra.run.smithi083.stdout: "pool_max": 2, 2024-09-14T03:40:38.783 INFO:teuthology.orchestra.run.smithi083.stdout: "max_osd": 4, 2024-09-14T03:40:38.783 INFO:teuthology.orchestra.run.smithi083.stdout: "require_min_compat_client": "squid", 2024-09-14T03:40:38.783 INFO:teuthology.orchestra.run.smithi083.stdout: "min_compat_client": "jewel", 2024-09-14T03:40:38.783 INFO:teuthology.orchestra.run.smithi083.stdout: "require_osd_release": "squid", 2024-09-14T03:40:38.783 INFO:teuthology.orchestra.run.smithi083.stdout: "allow_crimson": false, 2024-09-14T03:40:38.783 INFO:teuthology.orchestra.run.smithi083.stdout: "pools": [ 2024-09-14T03:40:38.783 INFO:teuthology.orchestra.run.smithi083.stdout: { 2024-09-14T03:40:38.783 INFO:teuthology.orchestra.run.smithi083.stdout: "pool": 1, 2024-09-14T03:40:38.783 INFO:teuthology.orchestra.run.smithi083.stdout: "pool_name": ".mgr", 2024-09-14T03:40:38.783 INFO:teuthology.orchestra.run.smithi083.stdout: "create_time": "2024-09-14T03:38:56.922548+0000", 2024-09-14T03:40:38.784 INFO:teuthology.orchestra.run.smithi083.stdout: "flags": 1, 2024-09-14T03:40:38.784 INFO:teuthology.orchestra.run.smithi083.stdout: "flags_names": "hashpspool", 2024-09-14T03:40:38.784 INFO:teuthology.orchestra.run.smithi083.stdout: "type": 1, 2024-09-14T03:40:38.784 INFO:teuthology.orchestra.run.smithi083.stdout: "size": 2, 2024-09-14T03:40:38.784 INFO:teuthology.orchestra.run.smithi083.stdout: "min_size": 1, 2024-09-14T03:40:38.784 INFO:teuthology.orchestra.run.smithi083.stdout: "crush_rule": 0, 2024-09-14T03:40:38.784 INFO:teuthology.orchestra.run.smithi083.stdout: "peering_crush_bucket_count": 0, 2024-09-14T03:40:38.784 INFO:teuthology.orchestra.run.smithi083.stdout: "peering_crush_bucket_target": 0, 2024-09-14T03:40:38.784 INFO:teuthology.orchestra.run.smithi083.stdout: "peering_crush_bucket_barrier": 0, 2024-09-14T03:40:38.784 INFO:teuthology.orchestra.run.smithi083.stdout: "peering_crush_bucket_mandatory_member": 2147483647, 2024-09-14T03:40:38.784 INFO:teuthology.orchestra.run.smithi083.stdout: "is_stretch_pool": false, 2024-09-14T03:40:38.784 INFO:teuthology.orchestra.run.smithi083.stdout: "object_hash": 2, 2024-09-14T03:40:38.784 INFO:teuthology.orchestra.run.smithi083.stdout: "pg_autoscale_mode": "off", 2024-09-14T03:40:38.784 INFO:teuthology.orchestra.run.smithi083.stdout: "pg_num": 1, 2024-09-14T03:40:38.785 INFO:teuthology.orchestra.run.smithi083.stdout: "pg_placement_num": 1, 2024-09-14T03:40:38.785 INFO:teuthology.orchestra.run.smithi083.stdout: "pg_placement_num_target": 1, 2024-09-14T03:40:38.785 INFO:teuthology.orchestra.run.smithi083.stdout: "pg_num_target": 1, 2024-09-14T03:40:38.785 INFO:teuthology.orchestra.run.smithi083.stdout: "pg_num_pending": 1, 2024-09-14T03:40:38.785 INFO:teuthology.orchestra.run.smithi083.stdout: "last_pg_merge_meta": { 2024-09-14T03:40:38.785 INFO:teuthology.orchestra.run.smithi083.stdout: "source_pgid": "0.0", 2024-09-14T03:40:38.785 INFO:teuthology.orchestra.run.smithi083.stdout: "ready_epoch": 0, 2024-09-14T03:40:38.785 INFO:teuthology.orchestra.run.smithi083.stdout: "last_epoch_started": 0, 2024-09-14T03:40:38.785 INFO:teuthology.orchestra.run.smithi083.stdout: "last_epoch_clean": 0, 2024-09-14T03:40:38.785 INFO:teuthology.orchestra.run.smithi083.stdout: "source_version": "0'0", 2024-09-14T03:40:38.785 INFO:teuthology.orchestra.run.smithi083.stdout: "target_version": "0'0" 2024-09-14T03:40:38.785 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-14T03:40:38.785 INFO:teuthology.orchestra.run.smithi083.stdout: "last_change": "14", 2024-09-14T03:40:38.786 INFO:teuthology.orchestra.run.smithi083.stdout: "last_force_op_resend": "0", 2024-09-14T03:40:38.786 INFO:teuthology.orchestra.run.smithi083.stdout: "last_force_op_resend_prenautilus": "0", 2024-09-14T03:40:38.786 INFO:teuthology.orchestra.run.smithi083.stdout: "last_force_op_resend_preluminous": "0", 2024-09-14T03:40:38.786 INFO:teuthology.orchestra.run.smithi083.stdout: "auid": 0, 2024-09-14T03:40:38.786 INFO:teuthology.orchestra.run.smithi083.stdout: "snap_mode": "selfmanaged", 2024-09-14T03:40:38.786 INFO:teuthology.orchestra.run.smithi083.stdout: "snap_seq": 0, 2024-09-14T03:40:38.786 INFO:teuthology.orchestra.run.smithi083.stdout: "snap_epoch": 0, 2024-09-14T03:40:38.786 INFO:teuthology.orchestra.run.smithi083.stdout: "pool_snaps": [], 2024-09-14T03:40:38.786 INFO:teuthology.orchestra.run.smithi083.stdout: "removed_snaps": "[]", 2024-09-14T03:40:38.786 INFO:teuthology.orchestra.run.smithi083.stdout: "quota_max_bytes": 0, 2024-09-14T03:40:38.786 INFO:teuthology.orchestra.run.smithi083.stdout: "quota_max_objects": 0, 2024-09-14T03:40:38.786 INFO:teuthology.orchestra.run.smithi083.stdout: "tiers": [], 2024-09-14T03:40:38.786 INFO:teuthology.orchestra.run.smithi083.stdout: "tier_of": -1, 2024-09-14T03:40:38.787 INFO:teuthology.orchestra.run.smithi083.stdout: "read_tier": -1, 2024-09-14T03:40:38.787 INFO:teuthology.orchestra.run.smithi083.stdout: "write_tier": -1, 2024-09-14T03:40:38.787 INFO:teuthology.orchestra.run.smithi083.stdout: "cache_mode": "none", 2024-09-14T03:40:38.787 INFO:teuthology.orchestra.run.smithi083.stdout: "target_max_bytes": 0, 2024-09-14T03:40:38.787 INFO:teuthology.orchestra.run.smithi083.stdout: "target_max_objects": 0, 2024-09-14T03:40:38.787 INFO:teuthology.orchestra.run.smithi083.stdout: "cache_target_dirty_ratio_micro": 400000, 2024-09-14T03:40:38.787 INFO:teuthology.orchestra.run.smithi083.stdout: "cache_target_dirty_high_ratio_micro": 600000, 2024-09-14T03:40:38.787 INFO:teuthology.orchestra.run.smithi083.stdout: "cache_target_full_ratio_micro": 800000, 2024-09-14T03:40:38.787 INFO:teuthology.orchestra.run.smithi083.stdout: "cache_min_flush_age": 0, 2024-09-14T03:40:38.787 INFO:teuthology.orchestra.run.smithi083.stdout: "cache_min_evict_age": 0, 2024-09-14T03:40:38.787 INFO:teuthology.orchestra.run.smithi083.stdout: "erasure_code_profile": "", 2024-09-14T03:40:38.787 INFO:teuthology.orchestra.run.smithi083.stdout: "hit_set_params": { 2024-09-14T03:40:38.787 INFO:teuthology.orchestra.run.smithi083.stdout: "type": "none" 2024-09-14T03:40:38.787 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-14T03:40:38.788 INFO:teuthology.orchestra.run.smithi083.stdout: "hit_set_period": 0, 2024-09-14T03:40:38.788 INFO:teuthology.orchestra.run.smithi083.stdout: "hit_set_count": 0, 2024-09-14T03:40:38.788 INFO:teuthology.orchestra.run.smithi083.stdout: "use_gmt_hitset": true, 2024-09-14T03:40:38.788 INFO:teuthology.orchestra.run.smithi083.stdout: "min_read_recency_for_promote": 0, 2024-09-14T03:40:38.788 INFO:teuthology.orchestra.run.smithi083.stdout: "min_write_recency_for_promote": 0, 2024-09-14T03:40:38.788 INFO:teuthology.orchestra.run.smithi083.stdout: "hit_set_grade_decay_rate": 0, 2024-09-14T03:40:38.788 INFO:teuthology.orchestra.run.smithi083.stdout: "hit_set_search_last_n": 0, 2024-09-14T03:40:38.788 INFO:teuthology.orchestra.run.smithi083.stdout: "grade_table": [], 2024-09-14T03:40:38.788 INFO:teuthology.orchestra.run.smithi083.stdout: "stripe_width": 0, 2024-09-14T03:40:38.788 INFO:teuthology.orchestra.run.smithi083.stdout: "expected_num_objects": 0, 2024-09-14T03:40:38.788 INFO:teuthology.orchestra.run.smithi083.stdout: "fast_read": false, 2024-09-14T03:40:38.788 INFO:teuthology.orchestra.run.smithi083.stdout: "options": { 2024-09-14T03:40:38.788 INFO:teuthology.orchestra.run.smithi083.stdout: "pg_num_max": 32, 2024-09-14T03:40:38.788 INFO:teuthology.orchestra.run.smithi083.stdout: "pg_num_min": 1 2024-09-14T03:40:38.789 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-14T03:40:38.789 INFO:teuthology.orchestra.run.smithi083.stdout: "application_metadata": { 2024-09-14T03:40:38.789 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": {} 2024-09-14T03:40:38.789 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-14T03:40:38.789 INFO:teuthology.orchestra.run.smithi083.stdout: "read_balance": { 2024-09-14T03:40:38.789 INFO:teuthology.orchestra.run.smithi083.stdout: "score_type": "Fair distribution", 2024-09-14T03:40:38.789 INFO:teuthology.orchestra.run.smithi083.stdout: "score_acting": 4, 2024-09-14T03:40:38.789 INFO:teuthology.orchestra.run.smithi083.stdout: "score_stable": 4, 2024-09-14T03:40:38.789 INFO:teuthology.orchestra.run.smithi083.stdout: "optimal_score": 0.5, 2024-09-14T03:40:38.789 INFO:teuthology.orchestra.run.smithi083.stdout: "raw_score_acting": 2, 2024-09-14T03:40:38.789 INFO:teuthology.orchestra.run.smithi083.stdout: "raw_score_stable": 2, 2024-09-14T03:40:38.789 INFO:teuthology.orchestra.run.smithi083.stdout: "primary_affinity_weighted": 1, 2024-09-14T03:40:38.789 INFO:teuthology.orchestra.run.smithi083.stdout: "average_primary_affinity": 1, 2024-09-14T03:40:38.790 INFO:teuthology.orchestra.run.smithi083.stdout: "average_primary_affinity_weighted": 1 2024-09-14T03:40:38.790 INFO:teuthology.orchestra.run.smithi083.stdout: } 2024-09-14T03:40:38.790 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-14T03:40:38.790 INFO:teuthology.orchestra.run.smithi083.stdout: { 2024-09-14T03:40:38.790 INFO:teuthology.orchestra.run.smithi083.stdout: "pool": 2, 2024-09-14T03:40:38.790 INFO:teuthology.orchestra.run.smithi083.stdout: "pool_name": "rbd", 2024-09-14T03:40:38.790 INFO:teuthology.orchestra.run.smithi083.stdout: "create_time": "2024-09-14T03:39:00.433011+0000", 2024-09-14T03:40:38.790 INFO:teuthology.orchestra.run.smithi083.stdout: "flags": 8193, 2024-09-14T03:40:38.790 INFO:teuthology.orchestra.run.smithi083.stdout: "flags_names": "hashpspool,selfmanaged_snaps", 2024-09-14T03:40:38.790 INFO:teuthology.orchestra.run.smithi083.stdout: "type": 1, 2024-09-14T03:40:38.791 INFO:teuthology.orchestra.run.smithi083.stdout: "size": 2, 2024-09-14T03:40:38.791 INFO:teuthology.orchestra.run.smithi083.stdout: "min_size": 1, 2024-09-14T03:40:38.791 INFO:teuthology.orchestra.run.smithi083.stdout: "crush_rule": 0, 2024-09-14T03:40:38.791 INFO:teuthology.orchestra.run.smithi083.stdout: "peering_crush_bucket_count": 0, 2024-09-14T03:40:38.791 INFO:teuthology.orchestra.run.smithi083.stdout: "peering_crush_bucket_target": 0, 2024-09-14T03:40:38.791 INFO:teuthology.orchestra.run.smithi083.stdout: "peering_crush_bucket_barrier": 0, 2024-09-14T03:40:38.791 INFO:teuthology.orchestra.run.smithi083.stdout: "peering_crush_bucket_mandatory_member": 2147483647, 2024-09-14T03:40:38.791 INFO:teuthology.orchestra.run.smithi083.stdout: "is_stretch_pool": false, 2024-09-14T03:40:38.791 INFO:teuthology.orchestra.run.smithi083.stdout: "object_hash": 2, 2024-09-14T03:40:38.791 INFO:teuthology.orchestra.run.smithi083.stdout: "pg_autoscale_mode": "off", 2024-09-14T03:40:38.791 INFO:teuthology.orchestra.run.smithi083.stdout: "pg_num": 8, 2024-09-14T03:40:38.792 INFO:teuthology.orchestra.run.smithi083.stdout: "pg_placement_num": 8, 2024-09-14T03:40:38.792 INFO:teuthology.orchestra.run.smithi083.stdout: "pg_placement_num_target": 8, 2024-09-14T03:40:38.792 INFO:teuthology.orchestra.run.smithi083.stdout: "pg_num_target": 8, 2024-09-14T03:40:38.792 INFO:teuthology.orchestra.run.smithi083.stdout: "pg_num_pending": 8, 2024-09-14T03:40:38.792 INFO:teuthology.orchestra.run.smithi083.stdout: "last_pg_merge_meta": { 2024-09-14T03:40:38.792 INFO:teuthology.orchestra.run.smithi083.stdout: "source_pgid": "0.0", 2024-09-14T03:40:38.792 INFO:teuthology.orchestra.run.smithi083.stdout: "ready_epoch": 0, 2024-09-14T03:40:38.792 INFO:teuthology.orchestra.run.smithi083.stdout: "last_epoch_started": 0, 2024-09-14T03:40:38.792 INFO:teuthology.orchestra.run.smithi083.stdout: "last_epoch_clean": 0, 2024-09-14T03:40:38.792 INFO:teuthology.orchestra.run.smithi083.stdout: "source_version": "0'0", 2024-09-14T03:40:38.792 INFO:teuthology.orchestra.run.smithi083.stdout: "target_version": "0'0" 2024-09-14T03:40:38.792 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-14T03:40:38.792 INFO:teuthology.orchestra.run.smithi083.stdout: "last_change": "18", 2024-09-14T03:40:38.792 INFO:teuthology.orchestra.run.smithi083.stdout: "last_force_op_resend": "0", 2024-09-14T03:40:38.793 INFO:teuthology.orchestra.run.smithi083.stdout: "last_force_op_resend_prenautilus": "0", 2024-09-14T03:40:38.793 INFO:teuthology.orchestra.run.smithi083.stdout: "last_force_op_resend_preluminous": "0", 2024-09-14T03:40:38.793 INFO:teuthology.orchestra.run.smithi083.stdout: "auid": 0, 2024-09-14T03:40:38.793 INFO:teuthology.orchestra.run.smithi083.stdout: "snap_mode": "selfmanaged", 2024-09-14T03:40:38.793 INFO:teuthology.orchestra.run.smithi083.stdout: "snap_seq": 2, 2024-09-14T03:40:38.793 INFO:teuthology.orchestra.run.smithi083.stdout: "snap_epoch": 18, 2024-09-14T03:40:38.793 INFO:teuthology.orchestra.run.smithi083.stdout: "pool_snaps": [], 2024-09-14T03:40:38.793 INFO:teuthology.orchestra.run.smithi083.stdout: "removed_snaps": "[]", 2024-09-14T03:40:38.793 INFO:teuthology.orchestra.run.smithi083.stdout: "quota_max_bytes": 0, 2024-09-14T03:40:38.793 INFO:teuthology.orchestra.run.smithi083.stdout: "quota_max_objects": 0, 2024-09-14T03:40:38.793 INFO:teuthology.orchestra.run.smithi083.stdout: "tiers": [], 2024-09-14T03:40:38.793 INFO:teuthology.orchestra.run.smithi083.stdout: "tier_of": -1, 2024-09-14T03:40:38.793 INFO:teuthology.orchestra.run.smithi083.stdout: "read_tier": -1, 2024-09-14T03:40:38.793 INFO:teuthology.orchestra.run.smithi083.stdout: "write_tier": -1, 2024-09-14T03:40:38.793 INFO:teuthology.orchestra.run.smithi083.stdout: "cache_mode": "none", 2024-09-14T03:40:38.794 INFO:teuthology.orchestra.run.smithi083.stdout: "target_max_bytes": 0, 2024-09-14T03:40:38.794 INFO:teuthology.orchestra.run.smithi083.stdout: "target_max_objects": 0, 2024-09-14T03:40:38.794 INFO:teuthology.orchestra.run.smithi083.stdout: "cache_target_dirty_ratio_micro": 400000, 2024-09-14T03:40:38.794 INFO:teuthology.orchestra.run.smithi083.stdout: "cache_target_dirty_high_ratio_micro": 600000, 2024-09-14T03:40:38.794 INFO:teuthology.orchestra.run.smithi083.stdout: "cache_target_full_ratio_micro": 800000, 2024-09-14T03:40:38.794 INFO:teuthology.orchestra.run.smithi083.stdout: "cache_min_flush_age": 0, 2024-09-14T03:40:38.794 INFO:teuthology.orchestra.run.smithi083.stdout: "cache_min_evict_age": 0, 2024-09-14T03:40:38.794 INFO:teuthology.orchestra.run.smithi083.stdout: "erasure_code_profile": "", 2024-09-14T03:40:38.794 INFO:teuthology.orchestra.run.smithi083.stdout: "hit_set_params": { 2024-09-14T03:40:38.794 INFO:teuthology.orchestra.run.smithi083.stdout: "type": "none" 2024-09-14T03:40:38.794 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-14T03:40:38.794 INFO:teuthology.orchestra.run.smithi083.stdout: "hit_set_period": 0, 2024-09-14T03:40:38.794 INFO:teuthology.orchestra.run.smithi083.stdout: "hit_set_count": 0, 2024-09-14T03:40:38.794 INFO:teuthology.orchestra.run.smithi083.stdout: "use_gmt_hitset": true, 2024-09-14T03:40:38.795 INFO:teuthology.orchestra.run.smithi083.stdout: "min_read_recency_for_promote": 0, 2024-09-14T03:40:38.795 INFO:teuthology.orchestra.run.smithi083.stdout: "min_write_recency_for_promote": 0, 2024-09-14T03:40:38.795 INFO:teuthology.orchestra.run.smithi083.stdout: "hit_set_grade_decay_rate": 0, 2024-09-14T03:40:38.795 INFO:teuthology.orchestra.run.smithi083.stdout: "hit_set_search_last_n": 0, 2024-09-14T03:40:38.795 INFO:teuthology.orchestra.run.smithi083.stdout: "grade_table": [], 2024-09-14T03:40:38.795 INFO:teuthology.orchestra.run.smithi083.stdout: "stripe_width": 0, 2024-09-14T03:40:38.795 INFO:teuthology.orchestra.run.smithi083.stdout: "expected_num_objects": 0, 2024-09-14T03:40:38.795 INFO:teuthology.orchestra.run.smithi083.stdout: "fast_read": false, 2024-09-14T03:40:38.795 INFO:teuthology.orchestra.run.smithi083.stdout: "options": {}, 2024-09-14T03:40:38.795 INFO:teuthology.orchestra.run.smithi083.stdout: "application_metadata": { 2024-09-14T03:40:38.796 INFO:teuthology.orchestra.run.smithi083.stdout: "rbd": {} 2024-09-14T03:40:38.796 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-14T03:40:38.796 INFO:teuthology.orchestra.run.smithi083.stdout: "read_balance": { 2024-09-14T03:40:38.796 INFO:teuthology.orchestra.run.smithi083.stdout: "score_type": "Fair distribution", 2024-09-14T03:40:38.796 INFO:teuthology.orchestra.run.smithi083.stdout: "score_acting": 2.5, 2024-09-14T03:40:38.796 INFO:teuthology.orchestra.run.smithi083.stdout: "score_stable": 2.5, 2024-09-14T03:40:38.796 INFO:teuthology.orchestra.run.smithi083.stdout: "optimal_score": 1, 2024-09-14T03:40:38.796 INFO:teuthology.orchestra.run.smithi083.stdout: "raw_score_acting": 2.5, 2024-09-14T03:40:38.796 INFO:teuthology.orchestra.run.smithi083.stdout: "raw_score_stable": 2.5, 2024-09-14T03:40:38.796 INFO:teuthology.orchestra.run.smithi083.stdout: "primary_affinity_weighted": 1, 2024-09-14T03:40:38.796 INFO:teuthology.orchestra.run.smithi083.stdout: "average_primary_affinity": 1, 2024-09-14T03:40:38.796 INFO:teuthology.orchestra.run.smithi083.stdout: "average_primary_affinity_weighted": 1 2024-09-14T03:40:38.796 INFO:teuthology.orchestra.run.smithi083.stdout: } 2024-09-14T03:40:38.796 INFO:teuthology.orchestra.run.smithi083.stdout: } 2024-09-14T03:40:38.797 INFO:teuthology.orchestra.run.smithi083.stdout: ], 2024-09-14T03:40:38.797 INFO:teuthology.orchestra.run.smithi083.stdout: "osds": [ 2024-09-14T03:40:38.797 INFO:teuthology.orchestra.run.smithi083.stdout: { 2024-09-14T03:40:38.797 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": 0, 2024-09-14T03:40:38.797 INFO:teuthology.orchestra.run.smithi083.stdout: "uuid": "ff477614-343d-4d97-9df5-94933f9b904d", 2024-09-14T03:40:38.797 INFO:teuthology.orchestra.run.smithi083.stdout: "up": 1, 2024-09-14T03:40:38.797 INFO:teuthology.orchestra.run.smithi083.stdout: "in": 1, 2024-09-14T03:40:38.797 INFO:teuthology.orchestra.run.smithi083.stdout: "weight": 1, 2024-09-14T03:40:38.797 INFO:teuthology.orchestra.run.smithi083.stdout: "primary_affinity": 1, 2024-09-14T03:40:38.797 INFO:teuthology.orchestra.run.smithi083.stdout: "last_clean_begin": 9, 2024-09-14T03:40:38.797 INFO:teuthology.orchestra.run.smithi083.stdout: "last_clean_end": 21, 2024-09-14T03:40:38.797 INFO:teuthology.orchestra.run.smithi083.stdout: "up_from": 27, 2024-09-14T03:40:38.797 INFO:teuthology.orchestra.run.smithi083.stdout: "up_thru": 27, 2024-09-14T03:40:38.797 INFO:teuthology.orchestra.run.smithi083.stdout: "down_at": 22, 2024-09-14T03:40:38.797 INFO:teuthology.orchestra.run.smithi083.stdout: "lost_at": 0, 2024-09-14T03:40:38.798 INFO:teuthology.orchestra.run.smithi083.stdout: "public_addrs": { 2024-09-14T03:40:38.798 INFO:teuthology.orchestra.run.smithi083.stdout: "addrvec": [ 2024-09-14T03:40:38.798 INFO:teuthology.orchestra.run.smithi083.stdout: { 2024-09-14T03:40:38.798 INFO:teuthology.orchestra.run.smithi083.stdout: "type": "v1", 2024-09-14T03:40:38.798 INFO:teuthology.orchestra.run.smithi083.stdout: "addr": "172.21.15.83:6801", 2024-09-14T03:40:38.798 INFO:teuthology.orchestra.run.smithi083.stdout: "nonce": 3543625573 2024-09-14T03:40:38.798 INFO:teuthology.orchestra.run.smithi083.stdout: } 2024-09-14T03:40:38.798 INFO:teuthology.orchestra.run.smithi083.stdout: ] 2024-09-14T03:40:38.798 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-14T03:40:38.798 INFO:teuthology.orchestra.run.smithi083.stdout: "cluster_addrs": { 2024-09-14T03:40:38.800 INFO:teuthology.orchestra.run.smithi083.stdout: "addrvec": [ 2024-09-14T03:40:38.800 INFO:teuthology.orchestra.run.smithi083.stdout: { 2024-09-14T03:40:38.800 INFO:teuthology.orchestra.run.smithi083.stdout: "type": "v1", 2024-09-14T03:40:38.800 INFO:teuthology.orchestra.run.smithi083.stdout: "addr": "172.21.15.83:6802", 2024-09-14T03:40:38.800 INFO:teuthology.orchestra.run.smithi083.stdout: "nonce": 3543625573 2024-09-14T03:40:38.800 INFO:teuthology.orchestra.run.smithi083.stdout: } 2024-09-14T03:40:38.800 INFO:teuthology.orchestra.run.smithi083.stdout: ] 2024-09-14T03:40:38.800 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-14T03:40:38.800 INFO:teuthology.orchestra.run.smithi083.stdout: "heartbeat_back_addrs": { 2024-09-14T03:40:38.800 INFO:teuthology.orchestra.run.smithi083.stdout: "addrvec": [ 2024-09-14T03:40:38.801 INFO:teuthology.orchestra.run.smithi083.stdout: { 2024-09-14T03:40:38.801 INFO:teuthology.orchestra.run.smithi083.stdout: "type": "v1", 2024-09-14T03:40:38.801 INFO:teuthology.orchestra.run.smithi083.stdout: "addr": "172.21.15.83:6804", 2024-09-14T03:40:38.801 INFO:teuthology.orchestra.run.smithi083.stdout: "nonce": 3543625573 2024-09-14T03:40:38.801 INFO:teuthology.orchestra.run.smithi083.stdout: } 2024-09-14T03:40:38.801 INFO:teuthology.orchestra.run.smithi083.stdout: ] 2024-09-14T03:40:38.801 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-14T03:40:38.801 INFO:teuthology.orchestra.run.smithi083.stdout: "heartbeat_front_addrs": { 2024-09-14T03:40:38.801 INFO:teuthology.orchestra.run.smithi083.stdout: "addrvec": [ 2024-09-14T03:40:38.801 INFO:teuthology.orchestra.run.smithi083.stdout: { 2024-09-14T03:40:38.801 INFO:teuthology.orchestra.run.smithi083.stdout: "type": "v1", 2024-09-14T03:40:38.801 INFO:teuthology.orchestra.run.smithi083.stdout: "addr": "172.21.15.83:6803", 2024-09-14T03:40:38.801 INFO:teuthology.orchestra.run.smithi083.stdout: "nonce": 3543625573 2024-09-14T03:40:38.802 INFO:teuthology.orchestra.run.smithi083.stdout: } 2024-09-14T03:40:38.802 INFO:teuthology.orchestra.run.smithi083.stdout: ] 2024-09-14T03:40:38.802 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-14T03:40:38.802 INFO:teuthology.orchestra.run.smithi083.stdout: "public_addr": "172.21.15.83:6801/3543625573", 2024-09-14T03:40:38.802 INFO:teuthology.orchestra.run.smithi083.stdout: "cluster_addr": "172.21.15.83:6802/3543625573", 2024-09-14T03:40:38.802 INFO:teuthology.orchestra.run.smithi083.stdout: "heartbeat_back_addr": "172.21.15.83:6804/3543625573", 2024-09-14T03:40:38.802 INFO:teuthology.orchestra.run.smithi083.stdout: "heartbeat_front_addr": "172.21.15.83:6803/3543625573", 2024-09-14T03:40:38.802 INFO:teuthology.orchestra.run.smithi083.stdout: "state": [ 2024-09-14T03:40:38.802 INFO:teuthology.orchestra.run.smithi083.stdout: "exists", 2024-09-14T03:40:38.802 INFO:teuthology.orchestra.run.smithi083.stdout: "up" 2024-09-14T03:40:38.802 INFO:teuthology.orchestra.run.smithi083.stdout: ] 2024-09-14T03:40:38.802 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-14T03:40:38.802 INFO:teuthology.orchestra.run.smithi083.stdout: { 2024-09-14T03:40:38.803 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": 1, 2024-09-14T03:40:38.803 INFO:teuthology.orchestra.run.smithi083.stdout: "uuid": "562ec9a0-3f91-4a29-896d-94340e9fa36f", 2024-09-14T03:40:38.803 INFO:teuthology.orchestra.run.smithi083.stdout: "up": 1, 2024-09-14T03:40:38.803 INFO:teuthology.orchestra.run.smithi083.stdout: "in": 1, 2024-09-14T03:40:38.803 INFO:teuthology.orchestra.run.smithi083.stdout: "weight": 1, 2024-09-14T03:40:38.803 INFO:teuthology.orchestra.run.smithi083.stdout: "primary_affinity": 1, 2024-09-14T03:40:38.803 INFO:teuthology.orchestra.run.smithi083.stdout: "last_clean_begin": 11, 2024-09-14T03:40:38.803 INFO:teuthology.orchestra.run.smithi083.stdout: "last_clean_end": 21, 2024-09-14T03:40:38.803 INFO:teuthology.orchestra.run.smithi083.stdout: "up_from": 27, 2024-09-14T03:40:38.803 INFO:teuthology.orchestra.run.smithi083.stdout: "up_thru": 27, 2024-09-14T03:40:38.803 INFO:teuthology.orchestra.run.smithi083.stdout: "down_at": 23, 2024-09-14T03:40:38.803 INFO:teuthology.orchestra.run.smithi083.stdout: "lost_at": 0, 2024-09-14T03:40:38.803 INFO:teuthology.orchestra.run.smithi083.stdout: "public_addrs": { 2024-09-14T03:40:38.803 INFO:teuthology.orchestra.run.smithi083.stdout: "addrvec": [ 2024-09-14T03:40:38.804 INFO:teuthology.orchestra.run.smithi083.stdout: { 2024-09-14T03:40:38.804 INFO:teuthology.orchestra.run.smithi083.stdout: "type": "v1", 2024-09-14T03:40:38.804 INFO:teuthology.orchestra.run.smithi083.stdout: "addr": "172.21.15.83:6805", 2024-09-14T03:40:38.804 INFO:teuthology.orchestra.run.smithi083.stdout: "nonce": 543231167 2024-09-14T03:40:38.804 INFO:teuthology.orchestra.run.smithi083.stdout: } 2024-09-14T03:40:38.804 INFO:teuthology.orchestra.run.smithi083.stdout: ] 2024-09-14T03:40:38.804 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-14T03:40:38.804 INFO:teuthology.orchestra.run.smithi083.stdout: "cluster_addrs": { 2024-09-14T03:40:38.804 INFO:teuthology.orchestra.run.smithi083.stdout: "addrvec": [ 2024-09-14T03:40:38.804 INFO:teuthology.orchestra.run.smithi083.stdout: { 2024-09-14T03:40:38.804 INFO:teuthology.orchestra.run.smithi083.stdout: "type": "v1", 2024-09-14T03:40:38.804 INFO:teuthology.orchestra.run.smithi083.stdout: "addr": "172.21.15.83:6806", 2024-09-14T03:40:38.804 INFO:teuthology.orchestra.run.smithi083.stdout: "nonce": 543231167 2024-09-14T03:40:38.804 INFO:teuthology.orchestra.run.smithi083.stdout: } 2024-09-14T03:40:38.805 INFO:teuthology.orchestra.run.smithi083.stdout: ] 2024-09-14T03:40:38.805 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-14T03:40:38.805 INFO:teuthology.orchestra.run.smithi083.stdout: "heartbeat_back_addrs": { 2024-09-14T03:40:38.805 INFO:teuthology.orchestra.run.smithi083.stdout: "addrvec": [ 2024-09-14T03:40:38.805 INFO:teuthology.orchestra.run.smithi083.stdout: { 2024-09-14T03:40:38.805 INFO:teuthology.orchestra.run.smithi083.stdout: "type": "v1", 2024-09-14T03:40:38.805 INFO:teuthology.orchestra.run.smithi083.stdout: "addr": "172.21.15.83:6808", 2024-09-14T03:40:38.805 INFO:teuthology.orchestra.run.smithi083.stdout: "nonce": 543231167 2024-09-14T03:40:38.805 INFO:teuthology.orchestra.run.smithi083.stdout: } 2024-09-14T03:40:38.805 INFO:teuthology.orchestra.run.smithi083.stdout: ] 2024-09-14T03:40:38.805 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-14T03:40:38.805 INFO:teuthology.orchestra.run.smithi083.stdout: "heartbeat_front_addrs": { 2024-09-14T03:40:38.805 INFO:teuthology.orchestra.run.smithi083.stdout: "addrvec": [ 2024-09-14T03:40:38.805 INFO:teuthology.orchestra.run.smithi083.stdout: { 2024-09-14T03:40:38.806 INFO:teuthology.orchestra.run.smithi083.stdout: "type": "v1", 2024-09-14T03:40:38.806 INFO:teuthology.orchestra.run.smithi083.stdout: "addr": "172.21.15.83:6807", 2024-09-14T03:40:38.806 INFO:teuthology.orchestra.run.smithi083.stdout: "nonce": 543231167 2024-09-14T03:40:38.806 INFO:teuthology.orchestra.run.smithi083.stdout: } 2024-09-14T03:40:38.806 INFO:teuthology.orchestra.run.smithi083.stdout: ] 2024-09-14T03:40:38.806 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-14T03:40:38.806 INFO:teuthology.orchestra.run.smithi083.stdout: "public_addr": "172.21.15.83:6805/543231167", 2024-09-14T03:40:38.806 INFO:teuthology.orchestra.run.smithi083.stdout: "cluster_addr": "172.21.15.83:6806/543231167", 2024-09-14T03:40:38.806 INFO:teuthology.orchestra.run.smithi083.stdout: "heartbeat_back_addr": "172.21.15.83:6808/543231167", 2024-09-14T03:40:38.806 INFO:teuthology.orchestra.run.smithi083.stdout: "heartbeat_front_addr": "172.21.15.83:6807/543231167", 2024-09-14T03:40:38.806 INFO:teuthology.orchestra.run.smithi083.stdout: "state": [ 2024-09-14T03:40:38.806 INFO:teuthology.orchestra.run.smithi083.stdout: "exists", 2024-09-14T03:40:38.806 INFO:teuthology.orchestra.run.smithi083.stdout: "up" 2024-09-14T03:40:38.806 INFO:teuthology.orchestra.run.smithi083.stdout: ] 2024-09-14T03:40:38.807 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-14T03:40:38.807 INFO:teuthology.orchestra.run.smithi083.stdout: { 2024-09-14T03:40:38.807 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": 2, 2024-09-14T03:40:38.807 INFO:teuthology.orchestra.run.smithi083.stdout: "uuid": "8c242dda-3308-4804-b2be-9a1737c42cf6", 2024-09-14T03:40:38.807 INFO:teuthology.orchestra.run.smithi083.stdout: "up": 1, 2024-09-14T03:40:38.807 INFO:teuthology.orchestra.run.smithi083.stdout: "in": 1, 2024-09-14T03:40:38.807 INFO:teuthology.orchestra.run.smithi083.stdout: "weight": 1, 2024-09-14T03:40:38.807 INFO:teuthology.orchestra.run.smithi083.stdout: "primary_affinity": 1, 2024-09-14T03:40:38.808 INFO:teuthology.orchestra.run.smithi083.stdout: "last_clean_begin": 11, 2024-09-14T03:40:38.808 INFO:teuthology.orchestra.run.smithi083.stdout: "last_clean_end": 23, 2024-09-14T03:40:38.809 INFO:teuthology.orchestra.run.smithi083.stdout: "up_from": 30, 2024-09-14T03:40:38.809 INFO:teuthology.orchestra.run.smithi083.stdout: "up_thru": 30, 2024-09-14T03:40:38.809 INFO:teuthology.orchestra.run.smithi083.stdout: "down_at": 24, 2024-09-14T03:40:38.809 INFO:teuthology.orchestra.run.smithi083.stdout: "lost_at": 0, 2024-09-14T03:40:38.809 INFO:teuthology.orchestra.run.smithi083.stdout: "public_addrs": { 2024-09-14T03:40:38.809 INFO:teuthology.orchestra.run.smithi083.stdout: "addrvec": [ 2024-09-14T03:40:38.809 INFO:teuthology.orchestra.run.smithi083.stdout: { 2024-09-14T03:40:38.809 INFO:teuthology.orchestra.run.smithi083.stdout: "type": "v1", 2024-09-14T03:40:38.809 INFO:teuthology.orchestra.run.smithi083.stdout: "addr": "172.21.15.83:6809", 2024-09-14T03:40:38.809 INFO:teuthology.orchestra.run.smithi083.stdout: "nonce": 905248239 2024-09-14T03:40:38.809 INFO:teuthology.orchestra.run.smithi083.stdout: } 2024-09-14T03:40:38.809 INFO:teuthology.orchestra.run.smithi083.stdout: ] 2024-09-14T03:40:38.810 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-14T03:40:38.810 INFO:teuthology.orchestra.run.smithi083.stdout: "cluster_addrs": { 2024-09-14T03:40:38.810 INFO:teuthology.orchestra.run.smithi083.stdout: "addrvec": [ 2024-09-14T03:40:38.810 INFO:teuthology.orchestra.run.smithi083.stdout: { 2024-09-14T03:40:38.810 INFO:teuthology.orchestra.run.smithi083.stdout: "type": "v1", 2024-09-14T03:40:38.810 INFO:teuthology.orchestra.run.smithi083.stdout: "addr": "172.21.15.83:6810", 2024-09-14T03:40:38.810 INFO:teuthology.orchestra.run.smithi083.stdout: "nonce": 905248239 2024-09-14T03:40:38.810 INFO:teuthology.orchestra.run.smithi083.stdout: } 2024-09-14T03:40:38.811 INFO:teuthology.orchestra.run.smithi083.stdout: ] 2024-09-14T03:40:38.811 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-14T03:40:38.811 INFO:teuthology.orchestra.run.smithi083.stdout: "heartbeat_back_addrs": { 2024-09-14T03:40:38.811 INFO:teuthology.orchestra.run.smithi083.stdout: "addrvec": [ 2024-09-14T03:40:38.811 INFO:teuthology.orchestra.run.smithi083.stdout: { 2024-09-14T03:40:38.811 INFO:teuthology.orchestra.run.smithi083.stdout: "type": "v1", 2024-09-14T03:40:38.811 INFO:teuthology.orchestra.run.smithi083.stdout: "addr": "172.21.15.83:6812", 2024-09-14T03:40:38.811 INFO:teuthology.orchestra.run.smithi083.stdout: "nonce": 905248239 2024-09-14T03:40:38.811 INFO:teuthology.orchestra.run.smithi083.stdout: } 2024-09-14T03:40:38.811 INFO:teuthology.orchestra.run.smithi083.stdout: ] 2024-09-14T03:40:38.811 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-14T03:40:38.811 INFO:teuthology.orchestra.run.smithi083.stdout: "heartbeat_front_addrs": { 2024-09-14T03:40:38.811 INFO:teuthology.orchestra.run.smithi083.stdout: "addrvec": [ 2024-09-14T03:40:38.811 INFO:teuthology.orchestra.run.smithi083.stdout: { 2024-09-14T03:40:38.812 INFO:teuthology.orchestra.run.smithi083.stdout: "type": "v1", 2024-09-14T03:40:38.812 INFO:teuthology.orchestra.run.smithi083.stdout: "addr": "172.21.15.83:6811", 2024-09-14T03:40:38.812 INFO:teuthology.orchestra.run.smithi083.stdout: "nonce": 905248239 2024-09-14T03:40:38.812 INFO:teuthology.orchestra.run.smithi083.stdout: } 2024-09-14T03:40:38.813 INFO:teuthology.orchestra.run.smithi083.stdout: ] 2024-09-14T03:40:38.813 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-14T03:40:38.813 INFO:teuthology.orchestra.run.smithi083.stdout: "public_addr": "172.21.15.83:6809/905248239", 2024-09-14T03:40:38.813 INFO:teuthology.orchestra.run.smithi083.stdout: "cluster_addr": "172.21.15.83:6810/905248239", 2024-09-14T03:40:38.813 INFO:teuthology.orchestra.run.smithi083.stdout: "heartbeat_back_addr": "172.21.15.83:6812/905248239", 2024-09-14T03:40:38.813 INFO:teuthology.orchestra.run.smithi083.stdout: "heartbeat_front_addr": "172.21.15.83:6811/905248239", 2024-09-14T03:40:38.813 INFO:teuthology.orchestra.run.smithi083.stdout: "state": [ 2024-09-14T03:40:38.813 INFO:teuthology.orchestra.run.smithi083.stdout: "exists", 2024-09-14T03:40:38.813 INFO:teuthology.orchestra.run.smithi083.stdout: "up" 2024-09-14T03:40:38.813 INFO:teuthology.orchestra.run.smithi083.stdout: ] 2024-09-14T03:40:38.813 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-14T03:40:38.813 INFO:teuthology.orchestra.run.smithi083.stdout: { 2024-09-14T03:40:38.813 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": 3, 2024-09-14T03:40:38.813 INFO:teuthology.orchestra.run.smithi083.stdout: "uuid": "c4dd2c36-3c86-4b56-86fb-fede641a436c", 2024-09-14T03:40:38.814 INFO:teuthology.orchestra.run.smithi083.stdout: "up": 1, 2024-09-14T03:40:38.814 INFO:teuthology.orchestra.run.smithi083.stdout: "in": 1, 2024-09-14T03:40:38.814 INFO:teuthology.orchestra.run.smithi083.stdout: "weight": 1, 2024-09-14T03:40:38.814 INFO:teuthology.orchestra.run.smithi083.stdout: "primary_affinity": 1, 2024-09-14T03:40:38.814 INFO:teuthology.orchestra.run.smithi083.stdout: "last_clean_begin": 11, 2024-09-14T03:40:38.814 INFO:teuthology.orchestra.run.smithi083.stdout: "last_clean_end": 24, 2024-09-14T03:40:38.814 INFO:teuthology.orchestra.run.smithi083.stdout: "up_from": 29, 2024-09-14T03:40:38.814 INFO:teuthology.orchestra.run.smithi083.stdout: "up_thru": 30, 2024-09-14T03:40:38.814 INFO:teuthology.orchestra.run.smithi083.stdout: "down_at": 25, 2024-09-14T03:40:38.815 INFO:teuthology.orchestra.run.smithi083.stdout: "lost_at": 0, 2024-09-14T03:40:38.815 INFO:teuthology.orchestra.run.smithi083.stdout: "public_addrs": { 2024-09-14T03:40:38.815 INFO:teuthology.orchestra.run.smithi083.stdout: "addrvec": [ 2024-09-14T03:40:38.815 INFO:teuthology.orchestra.run.smithi083.stdout: { 2024-09-14T03:40:38.815 INFO:teuthology.orchestra.run.smithi083.stdout: "type": "v1", 2024-09-14T03:40:38.815 INFO:teuthology.orchestra.run.smithi083.stdout: "addr": "172.21.15.83:6813", 2024-09-14T03:40:38.815 INFO:teuthology.orchestra.run.smithi083.stdout: "nonce": 3860913217 2024-09-14T03:40:38.815 INFO:teuthology.orchestra.run.smithi083.stdout: } 2024-09-14T03:40:38.815 INFO:teuthology.orchestra.run.smithi083.stdout: ] 2024-09-14T03:40:38.815 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-14T03:40:38.815 INFO:teuthology.orchestra.run.smithi083.stdout: "cluster_addrs": { 2024-09-14T03:40:38.816 INFO:teuthology.orchestra.run.smithi083.stdout: "addrvec": [ 2024-09-14T03:40:38.816 INFO:teuthology.orchestra.run.smithi083.stdout: { 2024-09-14T03:40:38.816 INFO:teuthology.orchestra.run.smithi083.stdout: "type": "v1", 2024-09-14T03:40:38.816 INFO:teuthology.orchestra.run.smithi083.stdout: "addr": "172.21.15.83:6814", 2024-09-14T03:40:38.816 INFO:teuthology.orchestra.run.smithi083.stdout: "nonce": 3860913217 2024-09-14T03:40:38.816 INFO:teuthology.orchestra.run.smithi083.stdout: } 2024-09-14T03:40:38.816 INFO:teuthology.orchestra.run.smithi083.stdout: ] 2024-09-14T03:40:38.816 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-14T03:40:38.816 INFO:teuthology.orchestra.run.smithi083.stdout: "heartbeat_back_addrs": { 2024-09-14T03:40:38.816 INFO:teuthology.orchestra.run.smithi083.stdout: "addrvec": [ 2024-09-14T03:40:38.816 INFO:teuthology.orchestra.run.smithi083.stdout: { 2024-09-14T03:40:38.816 INFO:teuthology.orchestra.run.smithi083.stdout: "type": "v1", 2024-09-14T03:40:38.816 INFO:teuthology.orchestra.run.smithi083.stdout: "addr": "172.21.15.83:6816", 2024-09-14T03:40:38.816 INFO:teuthology.orchestra.run.smithi083.stdout: "nonce": 3860913217 2024-09-14T03:40:38.817 INFO:teuthology.orchestra.run.smithi083.stdout: } 2024-09-14T03:40:38.817 INFO:teuthology.orchestra.run.smithi083.stdout: ] 2024-09-14T03:40:38.817 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-14T03:40:38.817 INFO:teuthology.orchestra.run.smithi083.stdout: "heartbeat_front_addrs": { 2024-09-14T03:40:38.817 INFO:teuthology.orchestra.run.smithi083.stdout: "addrvec": [ 2024-09-14T03:40:38.817 INFO:teuthology.orchestra.run.smithi083.stdout: { 2024-09-14T03:40:38.817 INFO:teuthology.orchestra.run.smithi083.stdout: "type": "v1", 2024-09-14T03:40:38.817 INFO:teuthology.orchestra.run.smithi083.stdout: "addr": "172.21.15.83:6815", 2024-09-14T03:40:38.817 INFO:teuthology.orchestra.run.smithi083.stdout: "nonce": 3860913217 2024-09-14T03:40:38.817 INFO:teuthology.orchestra.run.smithi083.stdout: } 2024-09-14T03:40:38.817 INFO:teuthology.orchestra.run.smithi083.stdout: ] 2024-09-14T03:40:38.817 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-14T03:40:38.817 INFO:teuthology.orchestra.run.smithi083.stdout: "public_addr": "172.21.15.83:6813/3860913217", 2024-09-14T03:40:38.817 INFO:teuthology.orchestra.run.smithi083.stdout: "cluster_addr": "172.21.15.83:6814/3860913217", 2024-09-14T03:40:38.817 INFO:teuthology.orchestra.run.smithi083.stdout: "heartbeat_back_addr": "172.21.15.83:6816/3860913217", 2024-09-14T03:40:38.818 INFO:teuthology.orchestra.run.smithi083.stdout: "heartbeat_front_addr": "172.21.15.83:6815/3860913217", 2024-09-14T03:40:38.818 INFO:teuthology.orchestra.run.smithi083.stdout: "state": [ 2024-09-14T03:40:38.818 INFO:teuthology.orchestra.run.smithi083.stdout: "exists", 2024-09-14T03:40:38.818 INFO:teuthology.orchestra.run.smithi083.stdout: "up" 2024-09-14T03:40:38.818 INFO:teuthology.orchestra.run.smithi083.stdout: ] 2024-09-14T03:40:38.818 INFO:teuthology.orchestra.run.smithi083.stdout: } 2024-09-14T03:40:38.818 INFO:teuthology.orchestra.run.smithi083.stdout: ], 2024-09-14T03:40:38.818 INFO:teuthology.orchestra.run.smithi083.stdout: "osd_xinfo": [ 2024-09-14T03:40:38.818 INFO:teuthology.orchestra.run.smithi083.stdout: { 2024-09-14T03:40:38.818 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": 0, 2024-09-14T03:40:38.818 INFO:teuthology.orchestra.run.smithi083.stdout: "down_stamp": "0.000000", 2024-09-14T03:40:38.818 INFO:teuthology.orchestra.run.smithi083.stdout: "laggy_probability": 0, 2024-09-14T03:40:38.818 INFO:teuthology.orchestra.run.smithi083.stdout: "laggy_interval": 0, 2024-09-14T03:40:38.818 INFO:teuthology.orchestra.run.smithi083.stdout: "features": 4540719139924082687, 2024-09-14T03:40:38.819 INFO:teuthology.orchestra.run.smithi083.stdout: "old_weight": 0, 2024-09-14T03:40:38.819 INFO:teuthology.orchestra.run.smithi083.stdout: "last_purged_snaps_scrub": "2024-09-14T03:38:53.047098+0000", 2024-09-14T03:40:38.819 INFO:teuthology.orchestra.run.smithi083.stdout: "dead_epoch": 22 2024-09-14T03:40:38.819 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-14T03:40:38.819 INFO:teuthology.orchestra.run.smithi083.stdout: { 2024-09-14T03:40:38.819 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": 1, 2024-09-14T03:40:38.819 INFO:teuthology.orchestra.run.smithi083.stdout: "down_stamp": "0.000000", 2024-09-14T03:40:38.819 INFO:teuthology.orchestra.run.smithi083.stdout: "laggy_probability": 0, 2024-09-14T03:40:38.819 INFO:teuthology.orchestra.run.smithi083.stdout: "laggy_interval": 0, 2024-09-14T03:40:38.819 INFO:teuthology.orchestra.run.smithi083.stdout: "features": 4540719139924082687, 2024-09-14T03:40:38.819 INFO:teuthology.orchestra.run.smithi083.stdout: "old_weight": 0, 2024-09-14T03:40:38.819 INFO:teuthology.orchestra.run.smithi083.stdout: "last_purged_snaps_scrub": "2024-09-14T03:38:53.354351+0000", 2024-09-14T03:40:38.819 INFO:teuthology.orchestra.run.smithi083.stdout: "dead_epoch": 23 2024-09-14T03:40:38.820 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-14T03:40:38.820 INFO:teuthology.orchestra.run.smithi083.stdout: { 2024-09-14T03:40:38.820 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": 2, 2024-09-14T03:40:38.820 INFO:teuthology.orchestra.run.smithi083.stdout: "down_stamp": "0.000000", 2024-09-14T03:40:38.820 INFO:teuthology.orchestra.run.smithi083.stdout: "laggy_probability": 0, 2024-09-14T03:40:38.820 INFO:teuthology.orchestra.run.smithi083.stdout: "laggy_interval": 0, 2024-09-14T03:40:38.820 INFO:teuthology.orchestra.run.smithi083.stdout: "features": 4540719139924082687, 2024-09-14T03:40:38.820 INFO:teuthology.orchestra.run.smithi083.stdout: "old_weight": 0, 2024-09-14T03:40:38.820 INFO:teuthology.orchestra.run.smithi083.stdout: "last_purged_snaps_scrub": "2024-09-14T03:38:53.422284+0000", 2024-09-14T03:40:38.820 INFO:teuthology.orchestra.run.smithi083.stdout: "dead_epoch": 24 2024-09-14T03:40:38.820 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-14T03:40:38.820 INFO:teuthology.orchestra.run.smithi083.stdout: { 2024-09-14T03:40:38.820 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": 3, 2024-09-14T03:40:38.820 INFO:teuthology.orchestra.run.smithi083.stdout: "down_stamp": "2024-09-14T03:40:28.062867+0000", 2024-09-14T03:40:38.821 INFO:teuthology.orchestra.run.smithi083.stdout: "laggy_probability": 0, 2024-09-14T03:40:38.821 INFO:teuthology.orchestra.run.smithi083.stdout: "laggy_interval": 0, 2024-09-14T03:40:38.821 INFO:teuthology.orchestra.run.smithi083.stdout: "features": 4540719139924082687, 2024-09-14T03:40:38.821 INFO:teuthology.orchestra.run.smithi083.stdout: "old_weight": 0, 2024-09-14T03:40:38.821 INFO:teuthology.orchestra.run.smithi083.stdout: "last_purged_snaps_scrub": "2024-09-14T03:38:53.620657+0000", 2024-09-14T03:40:38.821 INFO:teuthology.orchestra.run.smithi083.stdout: "dead_epoch": 0 2024-09-14T03:40:38.821 INFO:teuthology.orchestra.run.smithi083.stdout: } 2024-09-14T03:40:38.821 INFO:teuthology.orchestra.run.smithi083.stdout: ], 2024-09-14T03:40:38.821 INFO:teuthology.orchestra.run.smithi083.stdout: "pg_upmap": [], 2024-09-14T03:40:38.821 INFO:teuthology.orchestra.run.smithi083.stdout: "pg_upmap_items": [], 2024-09-14T03:40:38.821 INFO:teuthology.orchestra.run.smithi083.stdout: "pg_upmap_primaries": [], 2024-09-14T03:40:38.821 INFO:teuthology.orchestra.run.smithi083.stdout: "pg_temp": [], 2024-09-14T03:40:38.821 INFO:teuthology.orchestra.run.smithi083.stdout: "primary_temp": [], 2024-09-14T03:40:38.821 INFO:teuthology.orchestra.run.smithi083.stdout: "blocklist": { 2024-09-14T03:40:38.822 INFO:teuthology.orchestra.run.smithi083.stdout: "172.21.15.83:0/293392365": "2024-09-15T03:40:20.028874+0000", 2024-09-14T03:40:38.822 INFO:teuthology.orchestra.run.smithi083.stdout: "172.21.15.83:0/4246762177": "2024-09-15T03:40:20.028874+0000", 2024-09-14T03:40:38.822 INFO:teuthology.orchestra.run.smithi083.stdout: "172.21.15.83:0/3623382676": "2024-09-15T03:40:20.028874+0000", 2024-09-14T03:40:38.822 INFO:teuthology.orchestra.run.smithi083.stdout: "172.21.15.83:6800/1934165865": "2024-09-15T03:40:20.028874+0000", 2024-09-14T03:40:38.822 INFO:teuthology.orchestra.run.smithi083.stdout: "172.21.15.83:0/4208089382": "2024-09-15T03:40:20.028874+0000" 2024-09-14T03:40:38.822 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-14T03:40:38.822 INFO:teuthology.orchestra.run.smithi083.stdout: "range_blocklist": {}, 2024-09-14T03:40:38.822 INFO:teuthology.orchestra.run.smithi083.stdout: "erasure_code_profiles": { 2024-09-14T03:40:38.822 INFO:teuthology.orchestra.run.smithi083.stdout: "default": { 2024-09-14T03:40:38.822 INFO:teuthology.orchestra.run.smithi083.stdout: "crush-failure-domain": "osd", 2024-09-14T03:40:38.822 INFO:teuthology.orchestra.run.smithi083.stdout: "k": "2", 2024-09-14T03:40:38.822 INFO:teuthology.orchestra.run.smithi083.stdout: "m": "1", 2024-09-14T03:40:38.822 INFO:teuthology.orchestra.run.smithi083.stdout: "plugin": "jerasure", 2024-09-14T03:40:38.822 INFO:teuthology.orchestra.run.smithi083.stdout: "technique": "reed_sol_van" 2024-09-14T03:40:38.822 INFO:teuthology.orchestra.run.smithi083.stdout: } 2024-09-14T03:40:38.823 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-14T03:40:38.823 INFO:teuthology.orchestra.run.smithi083.stdout: "removed_snaps_queue": [], 2024-09-14T03:40:38.823 INFO:teuthology.orchestra.run.smithi083.stdout: "new_removed_snaps": [], 2024-09-14T03:40:38.823 INFO:teuthology.orchestra.run.smithi083.stdout: "new_purged_snaps": [], 2024-09-14T03:40:38.823 INFO:teuthology.orchestra.run.smithi083.stdout: "crush_node_flags": {}, 2024-09-14T03:40:38.823 INFO:teuthology.orchestra.run.smithi083.stdout: "device_class_flags": {}, 2024-09-14T03:40:38.823 INFO:teuthology.orchestra.run.smithi083.stdout: "stretch_mode": { 2024-09-14T03:40:38.823 INFO:teuthology.orchestra.run.smithi083.stdout: "stretch_mode_enabled": false, 2024-09-14T03:40:38.823 INFO:teuthology.orchestra.run.smithi083.stdout: "stretch_bucket_count": 0, 2024-09-14T03:40:38.823 INFO:teuthology.orchestra.run.smithi083.stdout: "degraded_stretch_mode": 0, 2024-09-14T03:40:38.823 INFO:teuthology.orchestra.run.smithi083.stdout: "recovering_stretch_mode": 0, 2024-09-14T03:40:38.823 INFO:teuthology.orchestra.run.smithi083.stdout: "stretch_mode_bucket": 0 2024-09-14T03:40:38.823 INFO:teuthology.orchestra.run.smithi083.stdout: } 2024-09-14T03:40:38.823 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-09-14T03:40:38.825 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.781+0000 7fb2b6227640 1 -- 172.21.15.83:0/3672203891 >> v1:172.21.15.83:6800/1803897559 conn(0x7fb29c0514e0 legacy=0x7fb29c0539a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:38.825 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.781+0000 7fb2b6227640 1 -- 172.21.15.83:0/3672203891 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fb2b00924c0 msgr2=0x7fb2b01c4dd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:38.825 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.781+0000 7fb2b6227640 1 --2- 172.21.15.83:0/3672203891 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fb2b00924c0 0x7fb2b01c4dd0 secure :-1 s=READY pgs=47 cs=0 l=1 rev1=1 crypto rx=0x7fb2a000ea60 tx=0x7fb2a000ef20 comp rx=0 tx=0).stop 2024-09-14T03:40:38.825 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.781+0000 7fb2b6227640 1 -- 172.21.15.83:0/3672203891 shutdown_connections 2024-09-14T03:40:38.825 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.781+0000 7fb2b6227640 1 --2- 172.21.15.83:0/3672203891 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fb2b012c180 0x7fb2b0072dc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:38.825 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.781+0000 7fb2b6227640 1 --2- 172.21.15.83:0/3672203891 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fb2b012b860 0x7fb2b01c5310 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:38.825 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.781+0000 7fb2b6227640 1 --2- 172.21.15.83:0/3672203891 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fb2b00924c0 0x7fb2b01c4dd0 unknown :-1 s=CLOSED pgs=47 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:38.825 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.781+0000 7fb2b6227640 1 -- 172.21.15.83:0/3672203891 >> 172.21.15.83:0/3672203891 conn(0x7fb2b009b580 msgr2=0x7fb2b0096430 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:38.826 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.781+0000 7fb2b6227640 1 -- 172.21.15.83:0/3672203891 shutdown_connections 2024-09-14T03:40:38.826 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.781+0000 7fb2b6227640 1 -- 172.21.15.83:0/3672203891 wait complete. 2024-09-14T03:40:38.826 DEBUG:teuthology.orchestra.run.smithi083:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'ceph fs dump' 2024-09-14T03:40:38.926 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.921+0000 7f23b3b3f640 1 -- 172.21.15.83:0/704101373 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f23ac12dea0 msgr2=0x7f23ac130290 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:38.926 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.921+0000 7f23b3b3f640 1 --2- 172.21.15.83:0/704101373 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f23ac12dea0 0x7f23ac130290 secure :-1 s=READY pgs=48 cs=0 l=1 rev1=1 crypto rx=0x7f239c009860 tx=0x7f239c02a460 comp rx=0 tx=0).stop 2024-09-14T03:40:38.926 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.921+0000 7f23b3b3f640 1 -- 172.21.15.83:0/704101373 shutdown_connections 2024-09-14T03:40:38.926 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.921+0000 7f23b3b3f640 1 --2- 172.21.15.83:0/704101373 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f23ac1307d0 0x7f23ac132bc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:38.926 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.921+0000 7f23b3b3f640 1 --2- 172.21.15.83:0/704101373 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f23ac12dea0 0x7f23ac130290 unknown :-1 s=CLOSED pgs=48 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:38.926 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.921+0000 7f23b3b3f640 1 -- 172.21.15.83:0/704101373 >> 172.21.15.83:0/704101373 conn(0x7f23ac09b580 msgr2=0x7f23ac09b990 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:38.926 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.921+0000 7f23b3b3f640 1 -- 172.21.15.83:0/704101373 shutdown_connections 2024-09-14T03:40:38.926 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.921+0000 7f23b3b3f640 1 -- 172.21.15.83:0/704101373 wait complete. 2024-09-14T03:40:38.926 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.922+0000 7f23b3b3f640 1 Processor -- start 2024-09-14T03:40:38.926 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.922+0000 7f23b3b3f640 1 -- start start 2024-09-14T03:40:38.926 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.922+0000 7f23b3b3f640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f23ac12b070 0x7f23ac08cd40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:38.926 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.922+0000 7f23b3b3f640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f23ac12dea0 0x7f23ac08d280 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:38.927 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.922+0000 7f23b3b3f640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f23ac1307d0 0x7f23ac08d7c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:38.927 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.922+0000 7f23b3b3f640 1 -- --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7f23ac1338a0 con 0x7f23ac12b070 2024-09-14T03:40:38.927 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.922+0000 7f23b3b3f640 1 -- --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7f23ac133720 con 0x7f23ac1307d0 2024-09-14T03:40:38.927 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.922+0000 7f23b3b3f640 1 -- --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7f23ac133a20 con 0x7f23ac12dea0 2024-09-14T03:40:38.927 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.923+0000 7f23b18db640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f23ac12b070 0x7f23ac08cd40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:38.927 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.923+0000 7f23b20dc640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f23ac1307d0 0x7f23ac08d7c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:38.927 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.923+0000 7f23b18db640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f23ac12b070 0x7f23ac08cd40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3300/0 says I am v2:172.21.15.83:37202/0 (socket says 172.21.15.83:37202) 2024-09-14T03:40:38.927 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.923+0000 7f23b18db640 1 -- 172.21.15.83:0/2479746388 learned_addr learned my addr 172.21.15.83:0/2479746388 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:40:38.927 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.923+0000 7f23b10da640 1 --2- 172.21.15.83:0/2479746388 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f23ac12dea0 0x7f23ac08d280 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:38.927 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.923+0000 7f23b18db640 1 -- 172.21.15.83:0/2479746388 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f23ac12dea0 msgr2=0x7f23ac08d280 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:38.927 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.923+0000 7f23b18db640 1 --2- 172.21.15.83:0/2479746388 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f23ac12dea0 0x7f23ac08d280 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:38.927 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.923+0000 7f23b18db640 1 -- 172.21.15.83:0/2479746388 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f23ac1307d0 msgr2=0x7f23ac08d7c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:38.927 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.923+0000 7f23b18db640 1 --2- 172.21.15.83:0/2479746388 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f23ac1307d0 0x7f23ac08d7c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:38.928 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.923+0000 7f23b18db640 1 -- 172.21.15.83:0/2479746388 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f23ac092f70 con 0x7f23ac12b070 2024-09-14T03:40:38.928 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.923+0000 7f23b10da640 1 --2- 172.21.15.83:0/2479746388 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f23ac12dea0 0x7f23ac08d280 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:40:38.928 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.923+0000 7f23b20dc640 1 --2- 172.21.15.83:0/2479746388 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f23ac1307d0 0x7f23ac08d7c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:40:38.928 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.923+0000 7f23b18db640 1 --2- 172.21.15.83:0/2479746388 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f23ac12b070 0x7f23ac08cd40 secure :-1 s=READY pgs=70 cs=0 l=1 rev1=1 crypto rx=0x7f23a800c490 tx=0x7f23a800c950 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:40:38.928 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.923+0000 7f23a27fc640 1 -- 172.21.15.83:0/2479746388 <== mon.0 v2:172.21.15.83:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f23a8016020 con 0x7f23ac12b070 2024-09-14T03:40:38.928 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.923+0000 7f23a27fc640 1 -- 172.21.15.83:0/2479746388 <== mon.0 v2:172.21.15.83:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f23a8012940 con 0x7f23ac12b070 2024-09-14T03:40:38.928 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.923+0000 7f23a27fc640 1 -- 172.21.15.83:0/2479746388 <== mon.0 v2:172.21.15.83:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f23a8012d90 con 0x7f23ac12b070 2024-09-14T03:40:38.928 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.923+0000 7f23b3b3f640 1 -- 172.21.15.83:0/2479746388 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f23ac08dd00 con 0x7f23ac12b070 2024-09-14T03:40:38.928 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.923+0000 7f23b3b3f640 1 -- 172.21.15.83:0/2479746388 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f23ac08f360 con 0x7f23ac12b070 2024-09-14T03:40:38.928 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.924+0000 7f23b3b3f640 1 -- 172.21.15.83:0/2479746388 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f23ac12b450 con 0x7f23ac12b070 2024-09-14T03:40:38.928 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.924+0000 7f23a27fc640 1 -- 172.21.15.83:0/2479746388 <== mon.0 v2:172.21.15.83:3300/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7f23a8019020 con 0x7f23ac12b070 2024-09-14T03:40:38.929 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.925+0000 7f23a27fc640 1 -- 172.21.15.83:0/2479746388 <== mon.0 v2:172.21.15.83:3300/0 5 ==== osd_map(31..31 src has 1..31) ==== 3027+0+0 (secure 0 0 0) 0x7f23a8068960 con 0x7f23ac12b070 2024-09-14T03:40:38.929 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:38.928+0000 7f23a27fc640 1 -- 172.21.15.83:0/2479746388 <== mon.0 v2:172.21.15.83:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+157506 (secure 0 0 0) 0x7f23a8020070 con 0x7f23ac12b070 2024-09-14T03:40:39.101 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.099+0000 7f23b3b3f640 1 -- 172.21.15.83:0/2479746388 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_command({"prefix": "fs dump"} v 0) -- 0x7f23ac074a10 con 0x7f23ac12b070 2024-09-14T03:40:39.101 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.100+0000 7f23a27fc640 1 -- 172.21.15.83:0/2479746388 <== mon.0 v2:172.21.15.83:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump"}]=0 dumped fsmap epoch 1 v1) ==== 75+0+440 (secure 0 0 0) 0x7f23a8067a10 con 0x7f23ac12b070 2024-09-14T03:40:39.102 INFO:teuthology.orchestra.run.smithi083.stdout:e1 2024-09-14T03:40:39.102 INFO:teuthology.orchestra.run.smithi083.stdout:btime 2024-09-14T03:38:43:953582+0000 2024-09-14T03:40:39.102 INFO:teuthology.orchestra.run.smithi083.stdout:enable_multiple, ever_enabled_multiple: 1,1 2024-09-14T03:40:39.102 INFO:teuthology.orchestra.run.smithi083.stdout:default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2,11=minor log segments,12=quiesce subvolumes} 2024-09-14T03:40:39.102 INFO:teuthology.orchestra.run.smithi083.stdout:legacy client fscid: -1 2024-09-14T03:40:39.102 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:40:39.102 INFO:teuthology.orchestra.run.smithi083.stdout:No filesystems configured 2024-09-14T03:40:39.102 INFO:teuthology.orchestra.run.smithi083.stderr:dumped fsmap epoch 1 2024-09-14T03:40:39.103 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.102+0000 7f23b3b3f640 1 -- 172.21.15.83:0/2479746388 >> v1:172.21.15.83:6800/1803897559 conn(0x7f23900514b0 legacy=0x7f2390053970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:39.103 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.102+0000 7f23b3b3f640 1 -- 172.21.15.83:0/2479746388 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f23ac12b070 msgr2=0x7f23ac08cd40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:39.104 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.102+0000 7f23b3b3f640 1 --2- 172.21.15.83:0/2479746388 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f23ac12b070 0x7f23ac08cd40 secure :-1 s=READY pgs=70 cs=0 l=1 rev1=1 crypto rx=0x7f23a800c490 tx=0x7f23a800c950 comp rx=0 tx=0).stop 2024-09-14T03:40:39.104 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.102+0000 7f23b3b3f640 1 -- 172.21.15.83:0/2479746388 shutdown_connections 2024-09-14T03:40:39.104 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.102+0000 7f23b3b3f640 1 --2- 172.21.15.83:0/2479746388 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f23ac1307d0 0x7f23ac08d7c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:39.104 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.102+0000 7f23b3b3f640 1 --2- 172.21.15.83:0/2479746388 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f23ac12dea0 0x7f23ac08d280 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:39.104 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.102+0000 7f23b3b3f640 1 --2- 172.21.15.83:0/2479746388 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f23ac12b070 0x7f23ac08cd40 unknown :-1 s=CLOSED pgs=70 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:39.104 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.102+0000 7f23b3b3f640 1 -- 172.21.15.83:0/2479746388 >> 172.21.15.83:0/2479746388 conn(0x7f23ac09b580 msgr2=0x7f23ac098ac0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:39.104 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.102+0000 7f23b3b3f640 1 -- 172.21.15.83:0/2479746388 shutdown_connections 2024-09-14T03:40:39.104 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.102+0000 7f23b3b3f640 1 -- 172.21.15.83:0/2479746388 wait complete. 2024-09-14T03:40:39.115 DEBUG:teuthology.orchestra.run.smithi083:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'ceph osd require-osd-release squid' 2024-09-14T03:40:39.212 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.210+0000 7fa5a01fe640 1 -- 172.21.15.83:0/3405338980 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fa598129d60 msgr2=0x7fa598133b60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:39.212 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.210+0000 7fa5a01fe640 1 --2- 172.21.15.83:0/3405338980 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fa598129d60 0x7fa598133b60 secure :-1 s=READY pgs=49 cs=0 l=1 rev1=1 crypto rx=0x7fa590005fd0 tx=0x7fa59002a980 comp rx=0 tx=0).stop 2024-09-14T03:40:39.212 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.210+0000 7fa5a01fe640 1 -- 172.21.15.83:0/3405338980 shutdown_connections 2024-09-14T03:40:39.212 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.210+0000 7fa5a01fe640 1 --2- 172.21.15.83:0/3405338980 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fa598129d60 0x7fa598133b60 unknown :-1 s=CLOSED pgs=49 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:39.212 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.210+0000 7fa5a01fe640 1 --2- 172.21.15.83:0/3405338980 >> v2:172.21.15.83:3300/0 conn(0x7fa598096350 0x7fa598096730 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:39.212 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.210+0000 7fa5a01fe640 1 -- 172.21.15.83:0/3405338980 >> 172.21.15.83:0/3405338980 conn(0x7fa59809a6e0 msgr2=0x7fa598097b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:39.212 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.210+0000 7fa5a01fe640 1 -- 172.21.15.83:0/3405338980 shutdown_connections 2024-09-14T03:40:39.212 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.210+0000 7fa5a01fe640 1 -- 172.21.15.83:0/3405338980 wait complete. 2024-09-14T03:40:39.214 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.211+0000 7fa5a01fe640 1 Processor -- start 2024-09-14T03:40:39.215 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.211+0000 7fa5a01fe640 1 -- start start 2024-09-14T03:40:39.215 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.211+0000 7fa5a01fe640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fa598096350 0x7fa598072890 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:39.215 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.211+0000 7fa5a01fe640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fa598096c70 0x7fa598072dd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:39.215 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.211+0000 7fa5a01fe640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fa598129d60 0x7fa598073310 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:39.215 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.211+0000 7fa5a01fe640 1 -- --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7fa598134620 con 0x7fa598096c70 2024-09-14T03:40:39.215 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.211+0000 7fa5a01fe640 1 -- --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7fa5981344a0 con 0x7fa598129d60 2024-09-14T03:40:39.215 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.211+0000 7fa5a01fe640 1 -- --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7fa5981347a0 con 0x7fa598096350 2024-09-14T03:40:39.215 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.211+0000 7fa59df9a640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fa598096350 0x7fa598072890 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:39.215 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.211+0000 7fa59d799640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fa598096c70 0x7fa598072dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:39.215 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.211+0000 7fa59e79b640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fa598129d60 0x7fa598073310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:39.215 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.212+0000 7fa59df9a640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fa598096350 0x7fa598072890 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3302/0 says I am v2:172.21.15.83:42366/0 (socket says 172.21.15.83:42366) 2024-09-14T03:40:39.215 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.212+0000 7fa59df9a640 1 -- 172.21.15.83:0/3982609932 learned_addr learned my addr 172.21.15.83:0/3982609932 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:40:39.215 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.212+0000 7fa59e79b640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fa598129d60 0x7fa598073310 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3301/0 says I am v2:172.21.15.83:52996/0 (socket says 172.21.15.83:52996) 2024-09-14T03:40:39.216 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.212+0000 7fa59d799640 1 -- 172.21.15.83:0/3982609932 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fa598096350 msgr2=0x7fa598072890 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:39.216 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.212+0000 7fa59d799640 1 --2- 172.21.15.83:0/3982609932 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fa598096350 0x7fa598072890 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:39.216 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.212+0000 7fa59d799640 1 -- 172.21.15.83:0/3982609932 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fa598129d60 msgr2=0x7fa598073310 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:39.216 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.212+0000 7fa59d799640 1 --2- 172.21.15.83:0/3982609932 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fa598129d60 0x7fa598073310 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:39.216 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.212+0000 7fa59d799640 1 -- 172.21.15.83:0/3982609932 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fa598095b90 con 0x7fa598096c70 2024-09-14T03:40:39.216 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.212+0000 7fa59df9a640 1 --2- 172.21.15.83:0/3982609932 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fa598096350 0x7fa598072890 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:40:39.216 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.212+0000 7fa59d799640 1 --2- 172.21.15.83:0/3982609932 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fa598096c70 0x7fa598072dd0 secure :-1 s=READY pgs=72 cs=0 l=1 rev1=1 crypto rx=0x7fa58800c430 tx=0x7fa58800c8f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:40:39.216 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.212+0000 7fa58e7fc640 1 -- 172.21.15.83:0/3982609932 <== mon.0 v2:172.21.15.83:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa588017070 con 0x7fa598096c70 2024-09-14T03:40:39.216 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.212+0000 7fa58e7fc640 1 -- 172.21.15.83:0/3982609932 <== mon.0 v2:172.21.15.83:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fa5880053c0 con 0x7fa598096c70 2024-09-14T03:40:39.216 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.212+0000 7fa5a01fe640 1 -- 172.21.15.83:0/3982609932 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fa598095110 con 0x7fa598096c70 2024-09-14T03:40:39.216 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.212+0000 7fa58e7fc640 1 -- 172.21.15.83:0/3982609932 <== mon.0 v2:172.21.15.83:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa588003850 con 0x7fa598096c70 2024-09-14T03:40:39.216 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.212+0000 7fa5a01fe640 1 -- 172.21.15.83:0/3982609932 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fa5981247e0 con 0x7fa598096c70 2024-09-14T03:40:39.216 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.213+0000 7fa5a01fe640 1 -- 172.21.15.83:0/3982609932 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fa598096730 con 0x7fa598096c70 2024-09-14T03:40:39.216 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.213+0000 7fa58e7fc640 1 -- 172.21.15.83:0/3982609932 <== mon.0 v2:172.21.15.83:3300/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7fa58801a070 con 0x7fa598096c70 2024-09-14T03:40:39.217 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.214+0000 7fa58e7fc640 1 -- 172.21.15.83:0/3982609932 <== mon.0 v2:172.21.15.83:3300/0 5 ==== osd_map(31..31 src has 1..31) ==== 3027+0+0 (secure 0 0 0) 0x7fa5880683b0 con 0x7fa598096c70 2024-09-14T03:40:39.218 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.217+0000 7fa58e7fc640 1 -- 172.21.15.83:0/3982609932 <== mon.0 v2:172.21.15.83:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+157506 (secure 0 0 0) 0x7fa588067460 con 0x7fa598096c70 2024-09-14T03:40:39.373 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.372+0000 7fa5a01fe640 1 -- 172.21.15.83:0/3982609932 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_command({"prefix": "osd require-osd-release", "release": "squid"} v 0) -- 0x7fa59806a3e0 con 0x7fa598096c70 2024-09-14T03:40:39.373 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.372+0000 7fa58e7fc640 1 -- 172.21.15.83:0/3982609932 <== mon.0 v2:172.21.15.83:3300/0 7 ==== mon_command_ack([{"prefix": "osd require-osd-release", "release": "squid"}]=0 v31) ==== 91+0+0 (secure 0 0 0) 0x7fa588066fa0 con 0x7fa598096c70 2024-09-14T03:40:39.375 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.374+0000 7fa5a01fe640 1 -- 172.21.15.83:0/3982609932 >> v1:172.21.15.83:6800/1803897559 conn(0x7fa57c0514b0 legacy=0x7fa57c053970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:39.375 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.374+0000 7fa5a01fe640 1 -- 172.21.15.83:0/3982609932 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fa598096c70 msgr2=0x7fa598072dd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:39.375 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.374+0000 7fa5a01fe640 1 --2- 172.21.15.83:0/3982609932 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fa598096c70 0x7fa598072dd0 secure :-1 s=READY pgs=72 cs=0 l=1 rev1=1 crypto rx=0x7fa58800c430 tx=0x7fa58800c8f0 comp rx=0 tx=0).stop 2024-09-14T03:40:39.376 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.374+0000 7fa5a01fe640 1 -- 172.21.15.83:0/3982609932 shutdown_connections 2024-09-14T03:40:39.376 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.374+0000 7fa5a01fe640 1 --2- 172.21.15.83:0/3982609932 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fa598129d60 0x7fa598073310 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:39.376 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.374+0000 7fa5a01fe640 1 --2- 172.21.15.83:0/3982609932 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fa598096c70 0x7fa598072dd0 unknown :-1 s=CLOSED pgs=72 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:39.376 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.374+0000 7fa5a01fe640 1 --2- 172.21.15.83:0/3982609932 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fa598096350 0x7fa598072890 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:39.376 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.374+0000 7fa5a01fe640 1 -- 172.21.15.83:0/3982609932 >> 172.21.15.83:0/3982609932 conn(0x7fa59809a6e0 msgr2=0x7fa59808ded0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:39.376 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.374+0000 7fa5a01fe640 1 -- 172.21.15.83:0/3982609932 shutdown_connections 2024-09-14T03:40:39.376 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.374+0000 7fa5a01fe640 1 -- 172.21.15.83:0/3982609932 wait complete. 2024-09-14T03:40:39.386 DEBUG:teuthology.orchestra.run.smithi083:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'for f in `ceph osd pool ls` ; do ceph osd pool set $f pg_autoscale_mode off ; done' 2024-09-14T03:40:39.483 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.481+0000 7f5b9349a640 1 -- 172.21.15.83:0/919315391 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f5b8c12b7a0 msgr2=0x7f5b8c12bc50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:39.484 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.481+0000 7f5b9349a640 1 --2- 172.21.15.83:0/919315391 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f5b8c12b7a0 0x7f5b8c12bc50 secure :-1 s=READY pgs=50 cs=0 l=1 rev1=1 crypto rx=0x7f5b84005fd0 tx=0x7f5b8402a8e0 comp rx=0 tx=0).stop 2024-09-14T03:40:39.484 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.482+0000 7f5b9349a640 1 -- 172.21.15.83:0/919315391 shutdown_connections 2024-09-14T03:40:39.484 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.482+0000 7f5b9349a640 1 --2- 172.21.15.83:0/919315391 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f5b8c12b7a0 0x7f5b8c12bc50 unknown :-1 s=CLOSED pgs=50 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:39.484 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.482+0000 7f5b9349a640 1 --2- 172.21.15.83:0/919315391 >> v2:172.21.15.83:3300/0 conn(0x7f5b8c0957c0 0x7f5b8c12b260 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:39.484 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.482+0000 7f5b9349a640 1 --2- 172.21.15.83:0/919315391 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f5b8c094ea0 0x7f5b8c095280 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:39.484 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.482+0000 7f5b9349a640 1 -- 172.21.15.83:0/919315391 >> 172.21.15.83:0/919315391 conn(0x7f5b8c09a6e0 msgr2=0x7f5b8c097b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:39.484 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.482+0000 7f5b9349a640 1 -- 172.21.15.83:0/919315391 shutdown_connections 2024-09-14T03:40:39.484 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.482+0000 7f5b9349a640 1 -- 172.21.15.83:0/919315391 wait complete. 2024-09-14T03:40:39.484 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.482+0000 7f5b9349a640 1 Processor -- start 2024-09-14T03:40:39.484 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.482+0000 7f5b9349a640 1 -- start start 2024-09-14T03:40:39.486 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.483+0000 7f5b9349a640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f5b8c094ea0 0x7f5b8c1303a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:39.486 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.483+0000 7f5b9349a640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f5b8c0957c0 0x7f5b8c1308e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:39.486 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.483+0000 7f5b9349a640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f5b8c12b7a0 0x7f5b8c130e20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:39.486 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.483+0000 7f5b9349a640 1 -- --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7f5b8c094ca0 con 0x7f5b8c094ea0 2024-09-14T03:40:39.487 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.483+0000 7f5b9349a640 1 -- --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7f5b8c094b20 con 0x7f5b8c0957c0 2024-09-14T03:40:39.487 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.483+0000 7f5b9349a640 1 -- --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7f5b8c132520 con 0x7f5b8c12b7a0 2024-09-14T03:40:39.487 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.483+0000 7f5b91236640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f5b8c094ea0 0x7f5b8c1303a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:39.487 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.483+0000 7f5b91a37640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f5b8c12b7a0 0x7f5b8c130e20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:39.487 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.483+0000 7f5b90a35640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f5b8c0957c0 0x7f5b8c1308e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:39.487 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.483+0000 7f5b91236640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f5b8c094ea0 0x7f5b8c1303a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3300/0 says I am v2:172.21.15.83:37252/0 (socket says 172.21.15.83:37252) 2024-09-14T03:40:39.487 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.483+0000 7f5b91236640 1 -- 172.21.15.83:0/4245478002 learned_addr learned my addr 172.21.15.83:0/4245478002 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:40:39.487 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.483+0000 7f5b90a35640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f5b8c0957c0 0x7f5b8c1308e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3301/0 says I am v2:172.21.15.83:53010/0 (socket says 172.21.15.83:53010) 2024-09-14T03:40:39.487 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.483+0000 7f5b91a37640 1 -- 172.21.15.83:0/4245478002 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f5b8c0957c0 msgr2=0x7f5b8c1308e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:39.487 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.483+0000 7f5b91a37640 1 --2- 172.21.15.83:0/4245478002 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f5b8c0957c0 0x7f5b8c1308e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:39.487 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.483+0000 7f5b91a37640 1 -- 172.21.15.83:0/4245478002 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f5b8c094ea0 msgr2=0x7f5b8c1303a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:39.487 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.483+0000 7f5b91a37640 1 --2- 172.21.15.83:0/4245478002 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f5b8c094ea0 0x7f5b8c1303a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:39.487 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.483+0000 7f5b91a37640 1 -- 172.21.15.83:0/4245478002 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f5b8c06ec50 con 0x7f5b8c12b7a0 2024-09-14T03:40:39.487 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.483+0000 7f5b91236640 1 --2- 172.21.15.83:0/4245478002 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f5b8c094ea0 0x7f5b8c1303a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:40:39.488 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.483+0000 7f5b91a37640 1 --2- 172.21.15.83:0/4245478002 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f5b8c12b7a0 0x7f5b8c130e20 secure :-1 s=READY pgs=49 cs=0 l=1 rev1=1 crypto rx=0x7f5b8c0960e0 tx=0x7f5b84003f10 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:40:39.488 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.483+0000 7f5b81ffb640 1 -- 172.21.15.83:0/4245478002 <== mon.2 v2:172.21.15.83:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5b84030ed0 con 0x7f5b8c12b7a0 2024-09-14T03:40:39.488 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.484+0000 7f5b81ffb640 1 -- 172.21.15.83:0/4245478002 <== mon.2 v2:172.21.15.83:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f5b84004040 con 0x7f5b8c12b7a0 2024-09-14T03:40:39.488 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.484+0000 7f5b81ffb640 1 -- 172.21.15.83:0/4245478002 <== mon.2 v2:172.21.15.83:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5b840044b0 con 0x7f5b8c12b7a0 2024-09-14T03:40:39.488 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.484+0000 7f5b9349a640 1 -- 172.21.15.83:0/4245478002 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f5b8c06fab0 con 0x7f5b8c12b7a0 2024-09-14T03:40:39.488 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.484+0000 7f5b9349a640 1 -- 172.21.15.83:0/4245478002 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7f5b8c06f140 con 0x7f5b8c12b7a0 2024-09-14T03:40:39.488 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.485+0000 7f5b9349a640 1 -- 172.21.15.83:0/4245478002 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f5b8c12b260 con 0x7f5b8c12b7a0 2024-09-14T03:40:39.488 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.485+0000 7f5b81ffb640 1 -- 172.21.15.83:0/4245478002 <== mon.2 v2:172.21.15.83:3302/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7f5b84004650 con 0x7f5b8c12b7a0 2024-09-14T03:40:39.488 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.485+0000 7f5b81ffb640 1 -- 172.21.15.83:0/4245478002 <== mon.2 v2:172.21.15.83:3302/0 5 ==== osd_map(31..31 src has 1..31) ==== 3027+0+0 (secure 0 0 0) 0x7f5b8408a560 con 0x7f5b8c12b7a0 2024-09-14T03:40:39.490 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.489+0000 7f5b81ffb640 1 -- 172.21.15.83:0/4245478002 <== mon.2 v2:172.21.15.83:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+157506 (secure 0 0 0) 0x7f5b84089610 con 0x7f5b8c12b7a0 2024-09-14T03:40:39.636 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.634+0000 7f5b9349a640 1 -- 172.21.15.83:0/4245478002 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_command({"prefix": "osd pool ls"} v 0) -- 0x7f5b8c06ee20 con 0x7f5b8c12b7a0 2024-09-14T03:40:39.636 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.635+0000 7f5b81ffb640 1 -- 172.21.15.83:0/4245478002 <== mon.2 v2:172.21.15.83:3302/0 7 ==== mon_command_ack([{"prefix": "osd pool ls"}]=0 v31) ==== 59+0+9 (secure 0 0 0) 0x7f5b84047da0 con 0x7f5b8c12b7a0 2024-09-14T03:40:39.638 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.637+0000 7f5b9349a640 1 -- 172.21.15.83:0/4245478002 >> v1:172.21.15.83:6800/1803897559 conn(0x7f5b700514e0 legacy=0x7f5b700539a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:39.638 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.637+0000 7f5b9349a640 1 -- 172.21.15.83:0/4245478002 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f5b8c12b7a0 msgr2=0x7f5b8c130e20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:39.638 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.637+0000 7f5b9349a640 1 --2- 172.21.15.83:0/4245478002 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f5b8c12b7a0 0x7f5b8c130e20 secure :-1 s=READY pgs=49 cs=0 l=1 rev1=1 crypto rx=0x7f5b8c0960e0 tx=0x7f5b84003f10 comp rx=0 tx=0).stop 2024-09-14T03:40:39.639 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.637+0000 7f5b9349a640 1 -- 172.21.15.83:0/4245478002 shutdown_connections 2024-09-14T03:40:39.639 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.637+0000 7f5b9349a640 1 --2- 172.21.15.83:0/4245478002 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f5b8c12b7a0 0x7f5b8c130e20 unknown :-1 s=CLOSED pgs=49 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:39.639 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.637+0000 7f5b9349a640 1 --2- 172.21.15.83:0/4245478002 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f5b8c0957c0 0x7f5b8c1308e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:39.639 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.637+0000 7f5b9349a640 1 --2- 172.21.15.83:0/4245478002 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f5b8c094ea0 0x7f5b8c1303a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:39.639 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.637+0000 7f5b9349a640 1 -- 172.21.15.83:0/4245478002 >> 172.21.15.83:0/4245478002 conn(0x7f5b8c09a6e0 msgr2=0x7f5b8c091ec0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:39.639 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.637+0000 7f5b9349a640 1 -- 172.21.15.83:0/4245478002 shutdown_connections 2024-09-14T03:40:39.639 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.637+0000 7f5b9349a640 1 -- 172.21.15.83:0/4245478002 wait complete. 2024-09-14T03:40:39.725 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.723+0000 7f8f697f9640 1 -- 172.21.15.83:0/4278870178 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f8f64095f50 msgr2=0x7f8f6412eb60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:39.725 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.723+0000 7f8f697f9640 1 --2- 172.21.15.83:0/4278870178 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f8f64095f50 0x7f8f6412eb60 secure :-1 s=READY pgs=50 cs=0 l=1 rev1=1 crypto rx=0x7f8f4c009830 tx=0x7f8f4c02a3f0 comp rx=0 tx=0).stop 2024-09-14T03:40:39.725 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.723+0000 7f8f697f9640 1 -- 172.21.15.83:0/4278870178 shutdown_connections 2024-09-14T03:40:39.725 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.723+0000 7f8f697f9640 1 --2- 172.21.15.83:0/4278870178 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f8f64095f50 0x7f8f6412eb60 unknown :-1 s=CLOSED pgs=50 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:39.725 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.723+0000 7f8f697f9640 1 --2- 172.21.15.83:0/4278870178 >> v2:172.21.15.83:3300/0 conn(0x7f8f64095630 0x7f8f64095a10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:39.726 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.723+0000 7f8f697f9640 1 -- 172.21.15.83:0/4278870178 >> 172.21.15.83:0/4278870178 conn(0x7f8f640a15c0 msgr2=0x7f8f640a19d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:39.726 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.723+0000 7f8f697f9640 1 -- 172.21.15.83:0/4278870178 shutdown_connections 2024-09-14T03:40:39.726 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.723+0000 7f8f697f9640 1 -- 172.21.15.83:0/4278870178 wait complete. 2024-09-14T03:40:39.726 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.724+0000 7f8f697f9640 1 Processor -- start 2024-09-14T03:40:39.726 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.724+0000 7f8f697f9640 1 -- start start 2024-09-14T03:40:39.728 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.724+0000 7f8f697f9640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f8f64095630 0x7f8f641c4dd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:39.728 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.724+0000 7f8f697f9640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f8f64095f50 0x7f8f641c5310 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:39.728 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.724+0000 7f8f697f9640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f8f6412f0a0 0x7f8f641c5850 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:39.728 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.724+0000 7f8f697f9640 1 -- --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7f8f64137980 con 0x7f8f64095f50 2024-09-14T03:40:39.728 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.724+0000 7f8f697f9640 1 -- --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7f8f64137800 con 0x7f8f64095630 2024-09-14T03:40:39.728 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.724+0000 7f8f697f9640 1 -- --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7f8f64137b00 con 0x7f8f6412f0a0 2024-09-14T03:40:39.729 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.724+0000 7f8f5bfff640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f8f64095f50 0x7f8f641c5310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:39.729 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.724+0000 7f8f62ffd640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f8f64095630 0x7f8f641c4dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:39.729 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.724+0000 7f8f637fe640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f8f6412f0a0 0x7f8f641c5850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:39.729 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.724+0000 7f8f5bfff640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f8f64095f50 0x7f8f641c5310 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3300/0 says I am v2:172.21.15.83:37272/0 (socket says 172.21.15.83:37272) 2024-09-14T03:40:39.729 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.724+0000 7f8f5bfff640 1 -- 172.21.15.83:0/358094256 learned_addr learned my addr 172.21.15.83:0/358094256 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:40:39.729 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.724+0000 7f8f62ffd640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f8f64095630 0x7f8f641c4dd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3301/0 says I am v2:172.21.15.83:53024/0 (socket says 172.21.15.83:53024) 2024-09-14T03:40:39.729 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.725+0000 7f8f5bfff640 1 -- 172.21.15.83:0/358094256 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f8f6412f0a0 msgr2=0x7f8f641c5850 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:39.729 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.725+0000 7f8f5bfff640 1 --2- 172.21.15.83:0/358094256 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f8f6412f0a0 0x7f8f641c5850 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:39.729 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.725+0000 7f8f5bfff640 1 -- 172.21.15.83:0/358094256 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f8f64095630 msgr2=0x7f8f641c4dd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:39.729 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.725+0000 7f8f5bfff640 1 --2- 172.21.15.83:0/358094256 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f8f64095630 0x7f8f641c4dd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:39.729 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.725+0000 7f8f5bfff640 1 -- 172.21.15.83:0/358094256 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f8f64091550 con 0x7f8f64095f50 2024-09-14T03:40:39.729 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.725+0000 7f8f62ffd640 1 --2- 172.21.15.83:0/358094256 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f8f64095630 0x7f8f641c4dd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:40:39.729 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.725+0000 7f8f5bfff640 1 --2- 172.21.15.83:0/358094256 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f8f64095f50 0x7f8f641c5310 secure :-1 s=READY pgs=74 cs=0 l=1 rev1=1 crypto rx=0x7f8f4c000c00 tx=0x7f8f4c032c10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:40:39.730 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.725+0000 7f8f5b7fe640 1 -- 172.21.15.83:0/358094256 <== mon.0 v2:172.21.15.83:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8f4c040070 con 0x7f8f64095f50 2024-09-14T03:40:39.730 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.725+0000 7f8f5b7fe640 1 -- 172.21.15.83:0/358094256 <== mon.0 v2:172.21.15.83:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f8f4c0038e0 con 0x7f8f64095f50 2024-09-14T03:40:39.730 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.725+0000 7f8f5b7fe640 1 -- 172.21.15.83:0/358094256 <== mon.0 v2:172.21.15.83:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8f4c003d30 con 0x7f8f64095f50 2024-09-14T03:40:39.730 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.725+0000 7f8f697f9640 1 -- 172.21.15.83:0/358094256 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f8f64132460 con 0x7f8f64095f50 2024-09-14T03:40:39.730 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.725+0000 7f8f697f9640 1 -- 172.21.15.83:0/358094256 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f8f640902d0 con 0x7f8f64095f50 2024-09-14T03:40:39.730 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.726+0000 7f8f5b7fe640 1 -- 172.21.15.83:0/358094256 <== mon.0 v2:172.21.15.83:3300/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7f8f4c039040 con 0x7f8f64095f50 2024-09-14T03:40:39.730 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.726+0000 7f8f697f9640 1 -- 172.21.15.83:0/358094256 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f8f6409c490 con 0x7f8f64095f50 2024-09-14T03:40:39.730 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.727+0000 7f8f5b7fe640 1 -- 172.21.15.83:0/358094256 <== mon.0 v2:172.21.15.83:3300/0 5 ==== osd_map(31..31 src has 1..31) ==== 3027+0+0 (secure 0 0 0) 0x7f8f4c08a3f0 con 0x7f8f64095f50 2024-09-14T03:40:39.731 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.730+0000 7f8f5b7fe640 1 -- 172.21.15.83:0/358094256 <== mon.0 v2:172.21.15.83:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+157506 (secure 0 0 0) 0x7f8f6409c490 con 0x7f8f64095f50 2024-09-14T03:40:39.877 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:39.875+0000 7f8f697f9640 1 -- 172.21.15.83:0/358094256 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_command({"prefix": "osd pool set", "pool": ".mgr", "var": "pg_autoscale_mode", "val": "off"} v 0) -- 0x7f8f64091a80 con 0x7f8f64095f50 2024-09-14T03:40:40.111 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:40.110+0000 7f8f5b7fe640 1 -- 172.21.15.83:0/358094256 <== mon.0 v2:172.21.15.83:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_autoscale_mode", "val": "off"}]=0 set pool 1 pg_autoscale_mode to off v32) ==== 153+0+0 (secure 0 0 0) 0x7f8f4c0894a0 con 0x7f8f64095f50 2024-09-14T03:40:40.111 INFO:teuthology.orchestra.run.smithi083.stderr:set pool 1 pg_autoscale_mode to off 2024-09-14T03:40:40.114 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:40.112+0000 7f8f697f9640 1 -- 172.21.15.83:0/358094256 >> v1:172.21.15.83:6800/1803897559 conn(0x7f8f440514b0 legacy=0x7f8f44053970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:40.114 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:40.112+0000 7f8f697f9640 1 -- 172.21.15.83:0/358094256 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f8f64095f50 msgr2=0x7f8f641c5310 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:40.114 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:40.112+0000 7f8f697f9640 1 --2- 172.21.15.83:0/358094256 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f8f64095f50 0x7f8f641c5310 secure :-1 s=READY pgs=74 cs=0 l=1 rev1=1 crypto rx=0x7f8f4c000c00 tx=0x7f8f4c032c10 comp rx=0 tx=0).stop 2024-09-14T03:40:40.114 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:40.112+0000 7f8f697f9640 1 -- 172.21.15.83:0/358094256 shutdown_connections 2024-09-14T03:40:40.114 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:40.112+0000 7f8f697f9640 1 --2- 172.21.15.83:0/358094256 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f8f6412f0a0 0x7f8f641c5850 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:40.115 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:40.112+0000 7f8f697f9640 1 --2- 172.21.15.83:0/358094256 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f8f64095f50 0x7f8f641c5310 unknown :-1 s=CLOSED pgs=74 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:40.115 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:40.112+0000 7f8f697f9640 1 --2- 172.21.15.83:0/358094256 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f8f64095630 0x7f8f641c4dd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:40.115 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:40.113+0000 7f8f697f9640 1 -- 172.21.15.83:0/358094256 >> 172.21.15.83:0/358094256 conn(0x7f8f640a15c0 msgr2=0x7f8f64098ac0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:40.115 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:40.113+0000 7f8f697f9640 1 -- 172.21.15.83:0/358094256 shutdown_connections 2024-09-14T03:40:40.115 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:40.113+0000 7f8f697f9640 1 -- 172.21.15.83:0/358094256 wait complete. 2024-09-14T03:40:40.202 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:40.200+0000 7fb82697b640 1 -- 172.21.15.83:0/1872064132 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fb82012bbb0 msgr2=0x7fb82012bf90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:40.202 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:40.200+0000 7fb82697b640 1 --2- 172.21.15.83:0/1872064132 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fb82012bbb0 0x7fb82012bf90 secure :-1 s=READY pgs=75 cs=0 l=1 rev1=1 crypto rx=0x7fb810005fd0 tx=0x7fb81002a410 comp rx=0 tx=0).stop 2024-09-14T03:40:40.202 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:40.200+0000 7fb82697b640 1 -- 172.21.15.83:0/1872064132 shutdown_connections 2024-09-14T03:40:40.202 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:40.200+0000 7fb82697b640 1 --2- 172.21.15.83:0/1872064132 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fb820094e90 0x7fb820096530 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:40.202 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:40.200+0000 7fb82697b640 1 --2- 172.21.15.83:0/1872064132 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fb820096a70 0x7fb820096f00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:40.203 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:40.200+0000 7fb82697b640 1 --2- 172.21.15.83:0/1872064132 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fb82012bbb0 0x7fb82012bf90 unknown :-1 s=CLOSED pgs=75 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:40.203 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:40.200+0000 7fb82697b640 1 -- 172.21.15.83:0/1872064132 >> 172.21.15.83:0/1872064132 conn(0x7fb82009d640 msgr2=0x7fb82009da50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:40.203 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:40.200+0000 7fb82697b640 1 -- 172.21.15.83:0/1872064132 shutdown_connections 2024-09-14T03:40:40.203 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:40.200+0000 7fb82697b640 1 -- 172.21.15.83:0/1872064132 wait complete. 2024-09-14T03:40:40.205 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:40.201+0000 7fb82697b640 1 Processor -- start 2024-09-14T03:40:40.205 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:40.201+0000 7fb82697b640 1 -- start start 2024-09-14T03:40:40.205 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:40.202+0000 7fb82697b640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fb820094e90 0x7fb8201c2ae0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:40.205 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:40.202+0000 7fb82697b640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fb820096a70 0x7fb8201c3020 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:40.205 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:40.202+0000 7fb82697b640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fb82012bbb0 0x7fb82012f920 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:40.206 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:40.202+0000 7fb82697b640 1 -- --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7fb820133470 con 0x7fb82012bbb0 2024-09-14T03:40:40.206 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:40.202+0000 7fb82697b640 1 -- --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7fb8201332f0 con 0x7fb820096a70 2024-09-14T03:40:40.206 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:40.202+0000 7fb82697b640 1 -- --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7fb8201335f0 con 0x7fb820094e90 2024-09-14T03:40:40.206 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:40.202+0000 7fb81ffff640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fb820094e90 0x7fb8201c2ae0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:40.206 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:40.202+0000 7fb81f7fe640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fb820096a70 0x7fb8201c3020 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:40.206 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:40.202+0000 7fb81ffff640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fb820094e90 0x7fb8201c2ae0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3302/0 says I am v2:172.21.15.83:42422/0 (socket says 172.21.15.83:42422) 2024-09-14T03:40:40.206 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:40.202+0000 7fb81ffff640 1 -- 172.21.15.83:0/254293364 learned_addr learned my addr 172.21.15.83:0/254293364 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:40:40.206 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:40.202+0000 7fb824f18640 1 --2- 172.21.15.83:0/254293364 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fb82012bbb0 0x7fb82012f920 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:40.206 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:40.202+0000 7fb81ffff640 1 -- 172.21.15.83:0/254293364 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fb820096a70 msgr2=0x7fb8201c3020 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:40.206 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:40.202+0000 7fb81ffff640 1 --2- 172.21.15.83:0/254293364 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fb820096a70 0x7fb8201c3020 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:40.206 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:40.202+0000 7fb81ffff640 1 -- 172.21.15.83:0/254293364 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fb82012bbb0 msgr2=0x7fb82012f920 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:40.206 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:40.202+0000 7fb81ffff640 1 --2- 172.21.15.83:0/254293364 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fb82012bbb0 0x7fb82012f920 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:40.206 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:40.202+0000 7fb81ffff640 1 -- 172.21.15.83:0/254293364 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fb820132300 con 0x7fb820094e90 2024-09-14T03:40:40.206 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:40.202+0000 7fb824f18640 1 --2- 172.21.15.83:0/254293364 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fb82012bbb0 0x7fb82012f920 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:40:40.207 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:40.202+0000 7fb81ffff640 1 --2- 172.21.15.83:0/254293364 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fb820094e90 0x7fb8201c2ae0 secure :-1 s=READY pgs=51 cs=0 l=1 rev1=1 crypto rx=0x7fb810009850 tx=0x7fb810005e80 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:40:40.207 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:40.202+0000 7fb81cff9640 1 -- 172.21.15.83:0/254293364 <== mon.2 v2:172.21.15.83:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb81003f070 con 0x7fb820094e90 2024-09-14T03:40:40.207 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:40.202+0000 7fb81cff9640 1 -- 172.21.15.83:0/254293364 <== mon.2 v2:172.21.15.83:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fb8100047a0 con 0x7fb820094e90 2024-09-14T03:40:40.207 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:40.202+0000 7fb82697b640 1 -- 172.21.15.83:0/254293364 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fb8200975f0 con 0x7fb820094e90 2024-09-14T03:40:40.207 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:40.202+0000 7fb81cff9640 1 -- 172.21.15.83:0/254293364 <== mon.2 v2:172.21.15.83:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb810035030 con 0x7fb820094e90 2024-09-14T03:40:40.207 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:40.203+0000 7fb82697b640 1 -- 172.21.15.83:0/254293364 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7fb82012fe60 con 0x7fb820094e90 2024-09-14T03:40:40.207 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:40.203+0000 7fb82697b640 1 -- 172.21.15.83:0/254293364 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fb820096530 con 0x7fb820094e90 2024-09-14T03:40:40.207 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:40.204+0000 7fb81cff9640 1 -- 172.21.15.83:0/254293364 <== mon.2 v2:172.21.15.83:3302/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7fb810042070 con 0x7fb820094e90 2024-09-14T03:40:40.207 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:40.204+0000 7fb81cff9640 1 -- 172.21.15.83:0/254293364 <== mon.2 v2:172.21.15.83:3302/0 5 ==== osd_map(32..32 src has 1..32) ==== 3027+0+0 (secure 0 0 0) 0x7fb810089d20 con 0x7fb820094e90 2024-09-14T03:40:40.209 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:40.207+0000 7fb81cff9640 1 -- 172.21.15.83:0/254293364 <== mon.2 v2:172.21.15.83:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+157506 (secure 0 0 0) 0x7fb8200975f0 con 0x7fb820094e90 2024-09-14T03:40:40.355 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:40.354+0000 7fb82697b640 1 -- 172.21.15.83:0/254293364 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_command({"prefix": "osd pool set", "pool": "rbd", "var": "pg_autoscale_mode", "val": "off"} v 0) -- 0x7fb82006a3e0 con 0x7fb820094e90 2024-09-14T03:40:41.121 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.119+0000 7fb81cff9640 1 -- 172.21.15.83:0/254293364 <== mon.2 v2:172.21.15.83:3302/0 7 ==== mon_command_ack([{"prefix": "osd pool set", "pool": "rbd", "var": "pg_autoscale_mode", "val": "off"}]=0 set pool 2 pg_autoscale_mode to off v33) ==== 152+0+0 (secure 0 0 0) 0x7fb820132300 con 0x7fb820094e90 2024-09-14T03:40:41.121 INFO:teuthology.orchestra.run.smithi083.stderr:set pool 2 pg_autoscale_mode to off 2024-09-14T03:40:41.123 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.121+0000 7fb82697b640 1 -- 172.21.15.83:0/254293364 >> v1:172.21.15.83:6800/1803897559 conn(0x7fb80c0514b0 legacy=0x7fb80c053970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:41.123 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.121+0000 7fb82697b640 1 -- 172.21.15.83:0/254293364 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fb820094e90 msgr2=0x7fb8201c2ae0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:41.123 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.121+0000 7fb82697b640 1 --2- 172.21.15.83:0/254293364 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fb820094e90 0x7fb8201c2ae0 secure :-1 s=READY pgs=51 cs=0 l=1 rev1=1 crypto rx=0x7fb810009850 tx=0x7fb810005e80 comp rx=0 tx=0).stop 2024-09-14T03:40:41.123 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.121+0000 7fb82697b640 1 -- 172.21.15.83:0/254293364 shutdown_connections 2024-09-14T03:40:41.123 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.122+0000 7fb82697b640 1 --2- 172.21.15.83:0/254293364 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fb82012bbb0 0x7fb82012f920 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:41.123 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.122+0000 7fb82697b640 1 --2- 172.21.15.83:0/254293364 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fb820096a70 0x7fb8201c3020 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:41.124 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.122+0000 7fb82697b640 1 --2- 172.21.15.83:0/254293364 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fb820094e90 0x7fb8201c2ae0 unknown :-1 s=CLOSED pgs=51 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:41.124 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.122+0000 7fb82697b640 1 -- 172.21.15.83:0/254293364 >> 172.21.15.83:0/254293364 conn(0x7fb82009d640 msgr2=0x7fb8200921a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:41.124 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.122+0000 7fb82697b640 1 -- 172.21.15.83:0/254293364 shutdown_connections 2024-09-14T03:40:41.124 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.122+0000 7fb82697b640 1 -- 172.21.15.83:0/254293364 wait complete. 2024-09-14T03:40:41.136 INFO:teuthology.run_tasks:Running task ceph.healthy... 2024-09-14T03:40:41.145 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-09-14T03:40:41.145 INFO:tasks.ceph.ceph_manager.ceph:waiting for mgr available 2024-09-14T03:40:41.145 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph mgr dump --format=json 2024-09-14T03:40:41.249 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.248+0000 7fdcfaaf2640 1 -- 172.21.15.83:0/1165815189 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fdcf41346f0 msgr2=0x7fdcf4136ae0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:41.249 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.248+0000 7fdcfaaf2640 1 --2- 172.21.15.83:0/1165815189 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fdcf41346f0 0x7fdcf4136ae0 secure :-1 s=READY pgs=77 cs=0 l=1 rev1=1 crypto rx=0x7fdcec009830 tx=0x7fdcec02a480 comp rx=0 tx=0).stop 2024-09-14T03:40:41.250 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.248+0000 7fdcfaaf2640 1 -- 172.21.15.83:0/1165815189 shutdown_connections 2024-09-14T03:40:41.250 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.248+0000 7fdcfaaf2640 1 --2- 172.21.15.83:0/1165815189 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fdcf41346f0 0x7fdcf4136ae0 unknown :-1 s=CLOSED pgs=77 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:41.250 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.248+0000 7fdcfaaf2640 1 --2- 172.21.15.83:0/1165815189 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fdcf4131dc0 0x7fdcf41341b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:41.250 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.248+0000 7fdcfaaf2640 1 -- 172.21.15.83:0/1165815189 >> 172.21.15.83:0/1165815189 conn(0x7fdcf40a1800 msgr2=0x7fdcf40a1c10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:41.250 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.248+0000 7fdcfaaf2640 1 -- 172.21.15.83:0/1165815189 shutdown_connections 2024-09-14T03:40:41.250 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.248+0000 7fdcfaaf2640 1 -- 172.21.15.83:0/1165815189 wait complete. 2024-09-14T03:40:41.252 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.248+0000 7fdcfaaf2640 1 Processor -- start 2024-09-14T03:40:41.252 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.249+0000 7fdcfaaf2640 1 -- start start 2024-09-14T03:40:41.252 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.249+0000 7fdcfaaf2640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fdcf4099080 0x7fdcf412de70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:41.252 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.249+0000 7fdcfaaf2640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fdcf4131dc0 0x7fdcf412e3b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:41.253 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.249+0000 7fdcfaaf2640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fdcf41346f0 0x7fdcf412e8f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:41.253 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.249+0000 7fdcfaaf2640 1 -- --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7fdcf41378d0 con 0x7fdcf4131dc0 2024-09-14T03:40:41.253 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.249+0000 7fdcfaaf2640 1 -- --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7fdcf4137750 con 0x7fdcf4099080 2024-09-14T03:40:41.253 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.249+0000 7fdcfaaf2640 1 -- --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7fdcf4137a50 con 0x7fdcf41346f0 2024-09-14T03:40:41.253 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.249+0000 7fdcf3fff640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fdcf4131dc0 0x7fdcf412e3b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:41.253 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.249+0000 7fdcf888e640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fdcf4099080 0x7fdcf412de70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:41.253 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.249+0000 7fdcf3fff640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fdcf4131dc0 0x7fdcf412e3b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3300/0 says I am v2:172.21.15.83:37302/0 (socket says 172.21.15.83:37302) 2024-09-14T03:40:41.253 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.249+0000 7fdcf3fff640 1 -- 172.21.15.83:0/534731079 learned_addr learned my addr 172.21.15.83:0/534731079 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:40:41.253 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.249+0000 7fdcf908f640 1 --2- 172.21.15.83:0/534731079 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fdcf41346f0 0x7fdcf412e8f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:41.253 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.249+0000 7fdcf3fff640 1 -- 172.21.15.83:0/534731079 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fdcf41346f0 msgr2=0x7fdcf412e8f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:41.253 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.249+0000 7fdcf3fff640 1 --2- 172.21.15.83:0/534731079 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fdcf41346f0 0x7fdcf412e8f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:41.253 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.250+0000 7fdcf3fff640 1 -- 172.21.15.83:0/534731079 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fdcf4099080 msgr2=0x7fdcf412de70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:41.253 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.250+0000 7fdcf3fff640 1 --2- 172.21.15.83:0/534731079 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fdcf4099080 0x7fdcf412de70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:41.253 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.250+0000 7fdcf3fff640 1 -- 172.21.15.83:0/534731079 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fdcf4097700 con 0x7fdcf4131dc0 2024-09-14T03:40:41.254 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.250+0000 7fdcf888e640 1 --2- 172.21.15.83:0/534731079 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fdcf4099080 0x7fdcf412de70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:40:41.254 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.250+0000 7fdcf3fff640 1 --2- 172.21.15.83:0/534731079 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fdcf4131dc0 0x7fdcf412e3b0 secure :-1 s=READY pgs=78 cs=0 l=1 rev1=1 crypto rx=0x7fdce4009d50 tx=0x7fdce400c6a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:40:41.254 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.250+0000 7fdcf17fa640 1 -- 172.21.15.83:0/534731079 <== mon.0 v2:172.21.15.83:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fdce4003b70 con 0x7fdcf4131dc0 2024-09-14T03:40:41.254 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.250+0000 7fdcf17fa640 1 -- 172.21.15.83:0/534731079 <== mon.0 v2:172.21.15.83:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fdce4003d10 con 0x7fdcf4131dc0 2024-09-14T03:40:41.254 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.250+0000 7fdcfaaf2640 1 -- 172.21.15.83:0/534731079 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fdcf412dbb0 con 0x7fdcf4131dc0 2024-09-14T03:40:41.254 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.250+0000 7fdcf17fa640 1 -- 172.21.15.83:0/534731079 <== mon.0 v2:172.21.15.83:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fdce4004280 con 0x7fdcf4131dc0 2024-09-14T03:40:41.254 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.250+0000 7fdcfaaf2640 1 -- 172.21.15.83:0/534731079 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fdcf412f440 con 0x7fdcf4131dc0 2024-09-14T03:40:41.254 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.251+0000 7fdcf17fa640 1 -- 172.21.15.83:0/534731079 <== mon.0 v2:172.21.15.83:3300/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7fdce4004420 con 0x7fdcf4131dc0 2024-09-14T03:40:41.254 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.251+0000 7fdcfaaf2640 1 -- 172.21.15.83:0/534731079 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fdcf412f6d0 con 0x7fdcf4131dc0 2024-09-14T03:40:41.254 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.251+0000 7fdcf17fa640 1 -- 172.21.15.83:0/534731079 <== mon.0 v2:172.21.15.83:3300/0 5 ==== osd_map(33..33 src has 1..33) ==== 3027+0+0 (secure 0 0 0) 0x7fdce4069cc0 con 0x7fdcf4131dc0 2024-09-14T03:40:41.256 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.255+0000 7fdcf17fa640 1 -- 172.21.15.83:0/534731079 <== mon.0 v2:172.21.15.83:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+157506 (secure 0 0 0) 0x7fdcf412f6d0 con 0x7fdcf4131dc0 2024-09-14T03:40:41.435 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.434+0000 7fdcfaaf2640 1 -- 172.21.15.83:0/534731079 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_command({"prefix": "mgr dump", "format": "json"} v 0) -- 0x7fdcf4099460 con 0x7fdcf4131dc0 2024-09-14T03:40:41.437 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.436+0000 7fdcf17fa640 1 -- 172.21.15.83:0/534731079 <== mon.0 v2:172.21.15.83:3300/0 7 ==== mon_command_ack([{"prefix": "mgr dump", "format": "json"}]=0 v10) ==== 74+0+124218 (secure 0 0 0) 0x7fdce4068d70 con 0x7fdcf4131dc0 2024-09-14T03:40:41.438 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:40:41.440 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.438+0000 7fdcfaaf2640 1 -- 172.21.15.83:0/534731079 >> v1:172.21.15.83:6800/1803897559 conn(0x7fdcd40514e0 legacy=0x7fdcd40539a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:41.440 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.438+0000 7fdcfaaf2640 1 -- 172.21.15.83:0/534731079 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fdcf4131dc0 msgr2=0x7fdcf412e3b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:41.440 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.438+0000 7fdcfaaf2640 1 --2- 172.21.15.83:0/534731079 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fdcf4131dc0 0x7fdcf412e3b0 secure :-1 s=READY pgs=78 cs=0 l=1 rev1=1 crypto rx=0x7fdce4009d50 tx=0x7fdce400c6a0 comp rx=0 tx=0).stop 2024-09-14T03:40:41.440 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.438+0000 7fdcfaaf2640 1 -- 172.21.15.83:0/534731079 shutdown_connections 2024-09-14T03:40:41.440 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.438+0000 7fdcfaaf2640 1 --2- 172.21.15.83:0/534731079 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fdcf41346f0 0x7fdcf412e8f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:41.440 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.438+0000 7fdcfaaf2640 1 --2- 172.21.15.83:0/534731079 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fdcf4131dc0 0x7fdcf412e3b0 unknown :-1 s=CLOSED pgs=78 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:41.440 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.438+0000 7fdcfaaf2640 1 --2- 172.21.15.83:0/534731079 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fdcf4099080 0x7fdcf412de70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:41.440 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.438+0000 7fdcfaaf2640 1 -- 172.21.15.83:0/534731079 >> 172.21.15.83:0/534731079 conn(0x7fdcf40a1800 msgr2=0x7fdcf4097b20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:41.440 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.438+0000 7fdcfaaf2640 1 -- 172.21.15.83:0/534731079 shutdown_connections 2024-09-14T03:40:41.440 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.438+0000 7fdcfaaf2640 1 -- 172.21.15.83:0/534731079 wait complete. 2024-09-14T03:40:41.450 INFO:teuthology.orchestra.run.smithi083.stdout:{"epoch":10,"flags":0,"active_gid":14106,"active_name":"y","active_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6800","nonce":1803897559}]},"active_addr":"172.21.15.83:6800/1803897559","active_change":"2024-09-14T03:40:20.029014+0000","active_mgr_features":4540719139924082687,"available":true,"standbys":[{"gid":14098,"name":"x","mgr_features":4540719139924082687,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to, use commas to separate multiple","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_requests":{"name":"max_requests","type":"int","level":"advanced","flags":0,"default_value":"500","min":"","max":"","enum_allowed":[],"desc":"Maximum number of requests to keep in memory. When new request comes in, the oldest request will be removed if the number of requests exceeds the max request number.if un-finished request is removed, error message will be logged in the ceph-mgr log.","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"secondary_zone_period_retry_limit":{"name":"secondary_zone_period_retry_limit","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"RGW module period update retry limit for secondary site","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"smb","can_run":true,"error_string":"","module_options":{"internal_store_backend":{"name":"internal_store_backend","type":"str","level":"dev","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"set internal store backend. for develoment and testing only","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"update_orchestration":{"name":"update_orchestration","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically update orchestration when smb resources are changed","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["iostat","nfs","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to, use commas to separate multiple","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_requests":{"name":"max_requests","type":"int","level":"advanced","flags":0,"default_value":"500","min":"","max":"","enum_allowed":[],"desc":"Maximum number of requests to keep in memory. When new request comes in, the oldest request will be removed if the number of requests exceeds the max request number.if un-finished request is removed, error message will be logged in the ceph-mgr log.","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"secondary_zone_period_retry_limit":{"name":"secondary_zone_period_retry_limit","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"RGW module period update retry limit for secondary site","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"smb","can_run":true,"error_string":"","module_options":{"internal_store_backend":{"name":"internal_store_backend","type":"str","level":"dev","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"set internal store backend. for develoment and testing only","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"update_orchestration":{"name":"update_orchestration","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically update orchestration when smb resources are changed","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"squid":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":21,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.83:0","nonce":3020827256}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.83:0","nonce":973842096}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.83:0","nonce":582793247}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.83:0","nonce":204434374}]}]} 2024-09-14T03:40:41.454 INFO:tasks.ceph.ceph_manager.ceph:mgr available! 2024-09-14T03:40:41.454 INFO:tasks.ceph.ceph_manager.ceph:waiting for all up 2024-09-14T03:40:41.454 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-09-14T03:40:41.554 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.552+0000 7faf7ceb3640 1 -- 172.21.15.83:0/2418927013 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7faf781309a0 msgr2=0x7faf78132d90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:41.554 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.552+0000 7faf7ceb3640 1 --2- 172.21.15.83:0/2418927013 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7faf781309a0 0x7faf78132d90 secure :-1 s=READY pgs=51 cs=0 l=1 rev1=1 crypto rx=0x7faf70005fd0 tx=0x7faf7002b860 comp rx=0 tx=0).stop 2024-09-14T03:40:41.554 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.552+0000 7faf7ceb3640 1 -- 172.21.15.83:0/2418927013 shutdown_connections 2024-09-14T03:40:41.554 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.552+0000 7faf7ceb3640 1 --2- 172.21.15.83:0/2418927013 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7faf781309a0 0x7faf78132d90 unknown :-1 s=CLOSED pgs=51 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:41.554 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.552+0000 7faf7ceb3640 1 --2- 172.21.15.83:0/2418927013 >> v2:172.21.15.83:3300/0 conn(0x7faf780955e0 0x7faf78130460 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:41.554 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.552+0000 7faf7ceb3640 1 -- 172.21.15.83:0/2418927013 >> 172.21.15.83:0/2418927013 conn(0x7faf7809a2e0 msgr2=0x7faf78097ad0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:41.555 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.552+0000 7faf7ceb3640 1 -- 172.21.15.83:0/2418927013 shutdown_connections 2024-09-14T03:40:41.555 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.552+0000 7faf7ceb3640 1 -- 172.21.15.83:0/2418927013 wait complete. 2024-09-14T03:40:41.557 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.553+0000 7faf7ceb3640 1 Processor -- start 2024-09-14T03:40:41.557 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.553+0000 7faf7ceb3640 1 -- start start 2024-09-14T03:40:41.558 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.553+0000 7faf7ceb3640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7faf780955e0 0x7faf781c2ab0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:41.558 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.553+0000 7faf7ceb3640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7faf7812b490 0x7faf781c2ff0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:41.558 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.554+0000 7faf7ceb3640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7faf781309a0 0x7faf781c3530 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:41.558 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.554+0000 7faf7ceb3640 1 -- --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7faf78133660 con 0x7faf780955e0 2024-09-14T03:40:41.558 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.554+0000 7faf7ceb3640 1 -- --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7faf781334e0 con 0x7faf7812b490 2024-09-14T03:40:41.558 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.554+0000 7faf7ceb3640 1 -- --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7faf781337e0 con 0x7faf781309a0 2024-09-14T03:40:41.558 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.554+0000 7faf6dd9b640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7faf7812b490 0x7faf781c2ff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:41.558 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.554+0000 7faf7659c640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7faf780955e0 0x7faf781c2ab0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:41.558 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.554+0000 7faf6dd9b640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7faf7812b490 0x7faf781c2ff0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3301/0 says I am v2:172.21.15.83:53068/0 (socket says 172.21.15.83:53068) 2024-09-14T03:40:41.558 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.554+0000 7faf6dd9b640 1 -- 172.21.15.83:0/3629364393 learned_addr learned my addr 172.21.15.83:0/3629364393 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:40:41.558 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.554+0000 7faf76d9d640 1 --2- 172.21.15.83:0/3629364393 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7faf781309a0 0x7faf781c3530 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:41.559 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.554+0000 7faf7659c640 1 -- 172.21.15.83:0/3629364393 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7faf781309a0 msgr2=0x7faf781c3530 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:41.559 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.554+0000 7faf7659c640 1 --2- 172.21.15.83:0/3629364393 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7faf781309a0 0x7faf781c3530 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:41.559 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.554+0000 7faf7659c640 1 -- 172.21.15.83:0/3629364393 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7faf7812b490 msgr2=0x7faf781c2ff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:41.559 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.554+0000 7faf7659c640 1 --2- 172.21.15.83:0/3629364393 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7faf7812b490 0x7faf781c2ff0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:41.559 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.554+0000 7faf7659c640 1 -- 172.21.15.83:0/3629364393 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7faf780908c0 con 0x7faf780955e0 2024-09-14T03:40:41.559 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.554+0000 7faf6dd9b640 1 --2- 172.21.15.83:0/3629364393 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7faf7812b490 0x7faf781c2ff0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:40:41.559 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.554+0000 7faf76d9d640 1 --2- 172.21.15.83:0/3629364393 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7faf781309a0 0x7faf781c3530 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:40:41.559 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.554+0000 7faf7659c640 1 --2- 172.21.15.83:0/3629364393 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7faf780955e0 0x7faf781c2ab0 secure :-1 s=READY pgs=80 cs=0 l=1 rev1=1 crypto rx=0x7faf6800c390 tx=0x7faf6800c850 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:40:41.559 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.554+0000 7faf6f7fe640 1 -- 172.21.15.83:0/3629364393 <== mon.0 v2:172.21.15.83:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7faf68017070 con 0x7faf780955e0 2024-09-14T03:40:41.559 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.554+0000 7faf6f7fe640 1 -- 172.21.15.83:0/3629364393 <== mon.0 v2:172.21.15.83:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7faf680035f0 con 0x7faf780955e0 2024-09-14T03:40:41.559 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.554+0000 7faf6f7fe640 1 -- 172.21.15.83:0/3629364393 <== mon.0 v2:172.21.15.83:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7faf68003a40 con 0x7faf780955e0 2024-09-14T03:40:41.560 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.554+0000 7faf7ceb3640 1 -- 172.21.15.83:0/3629364393 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7faf78090290 con 0x7faf780955e0 2024-09-14T03:40:41.560 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.554+0000 7faf7ceb3640 1 -- 172.21.15.83:0/3629364393 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7faf78070300 con 0x7faf780955e0 2024-09-14T03:40:41.560 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.555+0000 7faf7ceb3640 1 -- 172.21.15.83:0/3629364393 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7faf78098bc0 con 0x7faf780955e0 2024-09-14T03:40:41.560 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.555+0000 7faf6f7fe640 1 -- 172.21.15.83:0/3629364393 <== mon.0 v2:172.21.15.83:3300/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7faf6801a070 con 0x7faf780955e0 2024-09-14T03:40:41.560 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.556+0000 7faf6f7fe640 1 -- 172.21.15.83:0/3629364393 <== mon.0 v2:172.21.15.83:3300/0 5 ==== osd_map(33..33 src has 1..33) ==== 3027+0+0 (secure 0 0 0) 0x7faf680682c0 con 0x7faf780955e0 2024-09-14T03:40:41.561 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.559+0000 7faf6f7fe640 1 -- 172.21.15.83:0/3629364393 <== mon.0 v2:172.21.15.83:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+157506 (secure 0 0 0) 0x7faf78098bc0 con 0x7faf780955e0 2024-09-14T03:40:41.709 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.707+0000 7faf7ceb3640 1 -- 172.21.15.83:0/3629364393 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) -- 0x7faf78092a80 con 0x7faf780955e0 2024-09-14T03:40:41.709 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.708+0000 7faf6f7fe640 1 -- 172.21.15.83:0/3629364393 <== mon.0 v2:172.21.15.83:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v33) ==== 74+0+9162 (secure 0 0 0) 0x7faf78092a80 con 0x7faf780955e0 2024-09-14T03:40:41.709 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:40:41.710 INFO:teuthology.orchestra.run.smithi083.stdout:{"epoch":33,"fsid":"765298ef-df4c-4f28-af02-b45f60264ff9","created":"2024-09-14T03:38:43.954952+0000","modified":"2024-09-14T03:40:41.111243+0000","last_up_change":"2024-09-14T03:40:35.100253+0000","last_in_change":"2024-09-14T03:38:46.006167+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":8,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":4,"require_min_compat_client":"squid","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-14T03:38:56.922548+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"32","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":4,"score_stable":4,"optimal_score":0.5,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-14T03:39:00.433011+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"33","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":18,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.5,"score_stable":2.5,"optimal_score":1,"raw_score_acting":2.5,"raw_score_stable":2.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ff477614-343d-4d97-9df5-94933f9b904d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":9,"last_clean_end":21,"up_from":27,"up_thru":27,"down_at":22,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6801","nonce":3543625573}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6802","nonce":3543625573}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6804","nonce":3543625573}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6803","nonce":3543625573}]},"public_addr":"172.21.15.83:6801/3543625573","cluster_addr":"172.21.15.83:6802/3543625573","heartbeat_back_addr":"172.21.15.83:6804/3543625573","heartbeat_front_addr":"172.21.15.83:6803/3543625573","state":["exists","up"]},{"osd":1,"uuid":"562ec9a0-3f91-4a29-896d-94340e9fa36f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":11,"last_clean_end":21,"up_from":27,"up_thru":27,"down_at":23,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6805","nonce":543231167}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6806","nonce":543231167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6808","nonce":543231167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6807","nonce":543231167}]},"public_addr":"172.21.15.83:6805/543231167","cluster_addr":"172.21.15.83:6806/543231167","heartbeat_back_addr":"172.21.15.83:6808/543231167","heartbeat_front_addr":"172.21.15.83:6807/543231167","state":["exists","up"]},{"osd":2,"uuid":"8c242dda-3308-4804-b2be-9a1737c42cf6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":11,"last_clean_end":23,"up_from":30,"up_thru":30,"down_at":24,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6809","nonce":905248239}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6810","nonce":905248239}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6812","nonce":905248239}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6811","nonce":905248239}]},"public_addr":"172.21.15.83:6809/905248239","cluster_addr":"172.21.15.83:6810/905248239","heartbeat_back_addr":"172.21.15.83:6812/905248239","heartbeat_front_addr":"172.21.15.83:6811/905248239","state":["exists","up"]},{"osd":3,"uuid":"c4dd2c36-3c86-4b56-86fb-fede641a436c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":11,"last_clean_end":24,"up_from":29,"up_thru":30,"down_at":25,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6813","nonce":3860913217}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6814","nonce":3860913217}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6816","nonce":3860913217}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6815","nonce":3860913217}]},"public_addr":"172.21.15.83:6813/3860913217","cluster_addr":"172.21.15.83:6814/3860913217","heartbeat_back_addr":"172.21.15.83:6816/3860913217","heartbeat_front_addr":"172.21.15.83:6815/3860913217","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:38:53.047098+0000","dead_epoch":22},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:38:53.354351+0000","dead_epoch":23},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:38:53.422284+0000","dead_epoch":24},{"osd":3,"down_stamp":"2024-09-14T03:40:28.062867+0000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:38:53.620657+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.83:0/293392365":"2024-09-15T03:40:20.028874+0000","172.21.15.83:0/4246762177":"2024-09-15T03:40:20.028874+0000","172.21.15.83:0/3623382676":"2024-09-15T03:40:20.028874+0000","172.21.15.83:6800/1934165865":"2024-09-15T03:40:20.028874+0000","172.21.15.83:0/4208089382":"2024-09-15T03:40:20.028874+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-14T03:40:41.711 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.710+0000 7faf7ceb3640 1 -- 172.21.15.83:0/3629364393 >> v1:172.21.15.83:6800/1803897559 conn(0x7faf580514b0 legacy=0x7faf58053970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:41.712 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.710+0000 7faf7ceb3640 1 -- 172.21.15.83:0/3629364393 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7faf780955e0 msgr2=0x7faf781c2ab0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:41.712 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.710+0000 7faf7ceb3640 1 --2- 172.21.15.83:0/3629364393 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7faf780955e0 0x7faf781c2ab0 secure :-1 s=READY pgs=80 cs=0 l=1 rev1=1 crypto rx=0x7faf6800c390 tx=0x7faf6800c850 comp rx=0 tx=0).stop 2024-09-14T03:40:41.712 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.710+0000 7faf7ceb3640 1 -- 172.21.15.83:0/3629364393 shutdown_connections 2024-09-14T03:40:41.712 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.710+0000 7faf7ceb3640 1 --2- 172.21.15.83:0/3629364393 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7faf781309a0 0x7faf781c3530 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:41.712 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.710+0000 7faf7ceb3640 1 --2- 172.21.15.83:0/3629364393 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7faf7812b490 0x7faf781c2ff0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:41.712 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.710+0000 7faf7ceb3640 1 --2- 172.21.15.83:0/3629364393 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7faf780955e0 0x7faf781c2ab0 unknown :-1 s=CLOSED pgs=80 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:41.712 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.710+0000 7faf7ceb3640 1 -- 172.21.15.83:0/3629364393 >> 172.21.15.83:0/3629364393 conn(0x7faf7809a2e0 msgr2=0x7faf78091ea0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:41.712 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.710+0000 7faf7ceb3640 1 -- 172.21.15.83:0/3629364393 shutdown_connections 2024-09-14T03:40:41.712 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.710+0000 7faf7ceb3640 1 -- 172.21.15.83:0/3629364393 wait complete. 2024-09-14T03:40:41.722 INFO:tasks.ceph.ceph_manager.ceph:all up! 2024-09-14T03:40:41.722 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-09-14T03:40:41.823 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.821+0000 7fa4a0369640 1 -- 172.21.15.83:0/3483473199 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fa498095df0 msgr2=0x7fa498133b10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:41.823 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.821+0000 7fa4a0369640 1 --2- 172.21.15.83:0/3483473199 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fa498095df0 0x7fa498133b10 secure :-1 s=READY pgs=52 cs=0 l=1 rev1=1 crypto rx=0x7fa490009be0 tx=0x7fa49002a770 comp rx=0 tx=0).stop 2024-09-14T03:40:41.823 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.821+0000 7fa4a0369640 1 -- 172.21.15.83:0/3483473199 shutdown_connections 2024-09-14T03:40:41.823 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.821+0000 7fa4a0369640 1 --2- 172.21.15.83:0/3483473199 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fa498095df0 0x7fa498133b10 unknown :-1 s=CLOSED pgs=52 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:41.823 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.821+0000 7fa4a0369640 1 --2- 172.21.15.83:0/3483473199 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fa4980991f0 0x7fa49812bcc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:41.823 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.821+0000 7fa4a0369640 1 -- 172.21.15.83:0/3483473199 >> 172.21.15.83:0/3483473199 conn(0x7fa49809a2e0 msgr2=0x7fa498097ad0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:41.823 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.821+0000 7fa4a0369640 1 -- 172.21.15.83:0/3483473199 shutdown_connections 2024-09-14T03:40:41.823 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.821+0000 7fa4a0369640 1 -- 172.21.15.83:0/3483473199 wait complete. 2024-09-14T03:40:41.826 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.822+0000 7fa4a0369640 1 Processor -- start 2024-09-14T03:40:41.826 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.822+0000 7fa4a0369640 1 -- start start 2024-09-14T03:40:41.826 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.822+0000 7fa4a0369640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fa498095df0 0x7fa498122190 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:41.826 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.822+0000 7fa4a0369640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fa4980991f0 0x7fa4981226d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:41.826 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.822+0000 7fa4a0369640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fa49812c200 0x7fa498122c10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:41.826 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.822+0000 7fa4a0369640 1 -- --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7fa498134630 con 0x7fa498095df0 2024-09-14T03:40:41.826 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.822+0000 7fa4a0369640 1 -- --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7fa4981344b0 con 0x7fa49812c200 2024-09-14T03:40:41.826 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.822+0000 7fa4a0369640 1 -- --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7fa4981347b0 con 0x7fa4980991f0 2024-09-14T03:40:41.826 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.822+0000 7fa49e105640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fa498095df0 0x7fa498122190 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:41.826 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.822+0000 7fa49d904640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fa4980991f0 0x7fa4981226d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:41.826 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.822+0000 7fa49e105640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fa498095df0 0x7fa498122190 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3300/0 says I am v2:172.21.15.83:37336/0 (socket says 172.21.15.83:37336) 2024-09-14T03:40:41.826 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.822+0000 7fa49e105640 1 -- 172.21.15.83:0/1811006292 learned_addr learned my addr 172.21.15.83:0/1811006292 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:40:41.826 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.822+0000 7fa49e906640 1 --2- 172.21.15.83:0/1811006292 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fa49812c200 0x7fa498122c10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:41.827 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.823+0000 7fa49d904640 1 -- 172.21.15.83:0/1811006292 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fa49812c200 msgr2=0x7fa498122c10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:41.827 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.823+0000 7fa49d904640 1 --2- 172.21.15.83:0/1811006292 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fa49812c200 0x7fa498122c10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:41.827 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.823+0000 7fa49d904640 1 -- 172.21.15.83:0/1811006292 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fa498095df0 msgr2=0x7fa498122190 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:41.827 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.823+0000 7fa49d904640 1 --2- 172.21.15.83:0/1811006292 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fa498095df0 0x7fa498122190 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:41.827 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.823+0000 7fa49d904640 1 -- 172.21.15.83:0/1811006292 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fa498123f00 con 0x7fa4980991f0 2024-09-14T03:40:41.827 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.823+0000 7fa49e105640 1 --2- 172.21.15.83:0/1811006292 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fa498095df0 0x7fa498122190 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:40:41.827 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.823+0000 7fa49e906640 1 --2- 172.21.15.83:0/1811006292 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fa49812c200 0x7fa498122c10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:40:41.827 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.823+0000 7fa49d904640 1 --2- 172.21.15.83:0/1811006292 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fa4980991f0 0x7fa4981226d0 secure :-1 s=READY pgs=53 cs=0 l=1 rev1=1 crypto rx=0x7fa48800c3f0 tx=0x7fa48800c8b0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:40:41.827 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.823+0000 7fa48effd640 1 -- 172.21.15.83:0/1811006292 <== mon.2 v2:172.21.15.83:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa488011790 con 0x7fa4980991f0 2024-09-14T03:40:41.827 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.823+0000 7fa48effd640 1 -- 172.21.15.83:0/1811006292 <== mon.2 v2:172.21.15.83:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fa488017070 con 0x7fa4980991f0 2024-09-14T03:40:41.827 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.823+0000 7fa4a0369640 1 -- 172.21.15.83:0/1811006292 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fa498073d80 con 0x7fa4980991f0 2024-09-14T03:40:41.827 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.823+0000 7fa48effd640 1 -- 172.21.15.83:0/1811006292 <== mon.2 v2:172.21.15.83:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa488011ce0 con 0x7fa4980991f0 2024-09-14T03:40:41.827 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.823+0000 7fa4a0369640 1 -- 172.21.15.83:0/1811006292 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7fa498129040 con 0x7fa4980991f0 2024-09-14T03:40:41.827 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.824+0000 7fa4a0369640 1 -- 172.21.15.83:0/1811006292 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fa498074330 con 0x7fa4980991f0 2024-09-14T03:40:41.828 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.824+0000 7fa48effd640 1 -- 172.21.15.83:0/1811006292 <== mon.2 v2:172.21.15.83:3302/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7fa48801c460 con 0x7fa4980991f0 2024-09-14T03:40:41.828 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.825+0000 7fa48effd640 1 -- 172.21.15.83:0/1811006292 <== mon.2 v2:172.21.15.83:3302/0 5 ==== osd_map(33..33 src has 1..33) ==== 3027+0+0 (secure 0 0 0) 0x7fa488068cf0 con 0x7fa4980991f0 2024-09-14T03:40:41.829 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.828+0000 7fa48effd640 1 -- 172.21.15.83:0/1811006292 <== mon.2 v2:172.21.15.83:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+157506 (secure 0 0 0) 0x7fa498074330 con 0x7fa4980991f0 2024-09-14T03:40:41.977 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.975+0000 7fa4a0369640 1 -- 172.21.15.83:0/1811006292 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) -- 0x7fa49812bcc0 con 0x7fa4980991f0 2024-09-14T03:40:41.977 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.976+0000 7fa48effd640 1 -- 172.21.15.83:0/1811006292 <== mon.2 v2:172.21.15.83:3302/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v33) ==== 74+0+9162 (secure 0 0 0) 0x7fa48806d070 con 0x7fa4980991f0 2024-09-14T03:40:41.978 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:40:41.978 INFO:teuthology.orchestra.run.smithi083.stdout:{"epoch":33,"fsid":"765298ef-df4c-4f28-af02-b45f60264ff9","created":"2024-09-14T03:38:43.954952+0000","modified":"2024-09-14T03:40:41.111243+0000","last_up_change":"2024-09-14T03:40:35.100253+0000","last_in_change":"2024-09-14T03:38:46.006167+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":8,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":4,"require_min_compat_client":"squid","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-14T03:38:56.922548+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"32","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":4,"score_stable":4,"optimal_score":0.5,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-14T03:39:00.433011+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"33","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":18,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.5,"score_stable":2.5,"optimal_score":1,"raw_score_acting":2.5,"raw_score_stable":2.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ff477614-343d-4d97-9df5-94933f9b904d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":9,"last_clean_end":21,"up_from":27,"up_thru":27,"down_at":22,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6801","nonce":3543625573}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6802","nonce":3543625573}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6804","nonce":3543625573}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6803","nonce":3543625573}]},"public_addr":"172.21.15.83:6801/3543625573","cluster_addr":"172.21.15.83:6802/3543625573","heartbeat_back_addr":"172.21.15.83:6804/3543625573","heartbeat_front_addr":"172.21.15.83:6803/3543625573","state":["exists","up"]},{"osd":1,"uuid":"562ec9a0-3f91-4a29-896d-94340e9fa36f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":11,"last_clean_end":21,"up_from":27,"up_thru":27,"down_at":23,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6805","nonce":543231167}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6806","nonce":543231167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6808","nonce":543231167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6807","nonce":543231167}]},"public_addr":"172.21.15.83:6805/543231167","cluster_addr":"172.21.15.83:6806/543231167","heartbeat_back_addr":"172.21.15.83:6808/543231167","heartbeat_front_addr":"172.21.15.83:6807/543231167","state":["exists","up"]},{"osd":2,"uuid":"8c242dda-3308-4804-b2be-9a1737c42cf6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":11,"last_clean_end":23,"up_from":30,"up_thru":30,"down_at":24,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6809","nonce":905248239}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6810","nonce":905248239}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6812","nonce":905248239}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6811","nonce":905248239}]},"public_addr":"172.21.15.83:6809/905248239","cluster_addr":"172.21.15.83:6810/905248239","heartbeat_back_addr":"172.21.15.83:6812/905248239","heartbeat_front_addr":"172.21.15.83:6811/905248239","state":["exists","up"]},{"osd":3,"uuid":"c4dd2c36-3c86-4b56-86fb-fede641a436c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":11,"last_clean_end":24,"up_from":29,"up_thru":30,"down_at":25,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6813","nonce":3860913217}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6814","nonce":3860913217}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6816","nonce":3860913217}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6815","nonce":3860913217}]},"public_addr":"172.21.15.83:6813/3860913217","cluster_addr":"172.21.15.83:6814/3860913217","heartbeat_back_addr":"172.21.15.83:6816/3860913217","heartbeat_front_addr":"172.21.15.83:6815/3860913217","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:38:53.047098+0000","dead_epoch":22},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:38:53.354351+0000","dead_epoch":23},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:38:53.422284+0000","dead_epoch":24},{"osd":3,"down_stamp":"2024-09-14T03:40:28.062867+0000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:38:53.620657+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.83:0/293392365":"2024-09-15T03:40:20.028874+0000","172.21.15.83:0/4246762177":"2024-09-15T03:40:20.028874+0000","172.21.15.83:0/3623382676":"2024-09-15T03:40:20.028874+0000","172.21.15.83:6800/1934165865":"2024-09-15T03:40:20.028874+0000","172.21.15.83:0/4208089382":"2024-09-15T03:40:20.028874+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-14T03:40:41.980 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.978+0000 7fa4a0369640 1 -- 172.21.15.83:0/1811006292 >> v1:172.21.15.83:6800/1803897559 conn(0x7fa47c0514b0 legacy=0x7fa47c053970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:41.980 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.978+0000 7fa4a0369640 1 -- 172.21.15.83:0/1811006292 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fa4980991f0 msgr2=0x7fa4981226d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:41.981 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.978+0000 7fa4a0369640 1 --2- 172.21.15.83:0/1811006292 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fa4980991f0 0x7fa4981226d0 secure :-1 s=READY pgs=53 cs=0 l=1 rev1=1 crypto rx=0x7fa48800c3f0 tx=0x7fa48800c8b0 comp rx=0 tx=0).stop 2024-09-14T03:40:41.981 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.978+0000 7fa4a0369640 1 -- 172.21.15.83:0/1811006292 shutdown_connections 2024-09-14T03:40:41.981 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.978+0000 7fa4a0369640 1 --2- 172.21.15.83:0/1811006292 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fa49812c200 0x7fa498122c10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:41.981 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.978+0000 7fa4a0369640 1 --2- 172.21.15.83:0/1811006292 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fa4980991f0 0x7fa4981226d0 unknown :-1 s=CLOSED pgs=53 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:41.981 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.978+0000 7fa4a0369640 1 --2- 172.21.15.83:0/1811006292 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fa498095df0 0x7fa498122190 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:41.981 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.978+0000 7fa4a0369640 1 -- 172.21.15.83:0/1811006292 >> 172.21.15.83:0/1811006292 conn(0x7fa49809a2e0 msgr2=0x7fa4981295d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:41.981 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.978+0000 7fa4a0369640 1 -- 172.21.15.83:0/1811006292 shutdown_connections 2024-09-14T03:40:41.981 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:41.978+0000 7fa4a0369640 1 -- 172.21.15.83:0/1811006292 wait complete. 2024-09-14T03:40:41.991 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.0 flush_pg_stats 2024-09-14T03:40:41.991 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.1 flush_pg_stats 2024-09-14T03:40:41.991 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.2 flush_pg_stats 2024-09-14T03:40:41.992 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.3 flush_pg_stats 2024-09-14T03:40:42.099 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.098+0000 7efe4cb0f640 1 -- 172.21.15.83:0/890582884 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7efe48098d40 msgr2=0x7efe48099120 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:42.099 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.098+0000 7efe4cb0f640 1 --2- 172.21.15.83:0/890582884 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7efe48098d40 0x7efe48099120 secure :-1 s=READY pgs=54 cs=0 l=1 rev1=1 crypto rx=0x7efe38005fd0 tx=0x7efe3802a420 comp rx=0 tx=0).stop 2024-09-14T03:40:42.100 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.098+0000 7fa888803640 1 -- 172.21.15.83:0/124428880 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fa88412c770 msgr2=0x7fa884130f30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:42.100 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.098+0000 7efe4cb0f640 1 -- 172.21.15.83:0/890582884 shutdown_connections 2024-09-14T03:40:42.100 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.098+0000 7efe4cb0f640 1 --2- 172.21.15.83:0/890582884 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7efe4812eff0 0x7efe4812f480 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:42.100 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.098+0000 7efe4cb0f640 1 --2- 172.21.15.83:0/890582884 >> v2:172.21.15.83:3300/0 conn(0x7efe48099660 0x7efe4812eab0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:42.101 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.098+0000 7efe4cb0f640 1 --2- 172.21.15.83:0/890582884 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7efe48098d40 0x7efe48099120 unknown :-1 s=CLOSED pgs=54 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:42.101 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.098+0000 7efe4cb0f640 1 -- 172.21.15.83:0/890582884 >> 172.21.15.83:0/890582884 conn(0x7efe480a1790 msgr2=0x7efe480a1ba0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:42.101 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.098+0000 7fa888803640 1 --2- 172.21.15.83:0/124428880 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fa88412c770 0x7fa884130f30 secure :-1 s=READY pgs=52 cs=0 l=1 rev1=1 crypto rx=0x7fa878005fd0 tx=0x7fa87802a410 comp rx=0 tx=0).stop 2024-09-14T03:40:42.101 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.098+0000 7efe4cb0f640 1 -- 172.21.15.83:0/890582884 shutdown_connections 2024-09-14T03:40:42.102 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.098+0000 7fa888803640 1 -- 172.21.15.83:0/124428880 shutdown_connections 2024-09-14T03:40:42.102 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.098+0000 7fa888803640 1 --2- 172.21.15.83:0/124428880 >> v2:172.21.15.83:3300/0 conn(0x7fa884131470 0x7fa884133860 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:42.102 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.098+0000 7fa888803640 1 --2- 172.21.15.83:0/124428880 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fa88412c770 0x7fa884130f30 unknown :-1 s=CLOSED pgs=52 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:42.102 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.098+0000 7fa888803640 1 -- 172.21.15.83:0/124428880 >> 172.21.15.83:0/124428880 conn(0x7fa88409ba30 msgr2=0x7fa88409be40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:42.102 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.098+0000 7fa888803640 1 -- 172.21.15.83:0/124428880 shutdown_connections 2024-09-14T03:40:42.102 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.098+0000 7efc7a374640 1 -- 172.21.15.83:0/1962926637 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7efc740700b0 msgr2=0x7efc74070490 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:42.103 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.098+0000 7efc7a374640 1 --2- 172.21.15.83:0/1962926637 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7efc740700b0 0x7efc74070490 secure :-1 s=READY pgs=86 cs=0 l=1 rev1=1 crypto rx=0x7efc64005fd0 tx=0x7efc6402a4b0 comp rx=0 tx=0).stop 2024-09-14T03:40:42.103 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.098+0000 7fa888803640 1 -- 172.21.15.83:0/124428880 wait complete. 2024-09-14T03:40:42.103 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.098+0000 7efe4cb0f640 1 -- 172.21.15.83:0/890582884 wait complete. 2024-09-14T03:40:42.104 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.098+0000 7efc7a374640 1 -- 172.21.15.83:0/1962926637 shutdown_connections 2024-09-14T03:40:42.104 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.098+0000 7efc7a374640 1 --2- 172.21.15.83:0/1962926637 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7efc740709d0 0x7efc7412bbf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:42.104 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.098+0000 7efc7a374640 1 --2- 172.21.15.83:0/1962926637 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7efc740700b0 0x7efc74070490 unknown :-1 s=CLOSED pgs=86 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:42.104 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.098+0000 7efc7a374640 1 -- 172.21.15.83:0/1962926637 >> 172.21.15.83:0/1962926637 conn(0x7efc7409b9a0 msgr2=0x7efc7409bd80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:42.104 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.098+0000 7efc7a374640 1 -- 172.21.15.83:0/1962926637 shutdown_connections 2024-09-14T03:40:42.104 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.098+0000 7efc7a374640 1 -- 172.21.15.83:0/1962926637 wait complete. 2024-09-14T03:40:42.105 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.099+0000 7f76949c5640 1 -- 172.21.15.83:0/45398672 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f769012f1c0 msgr2=0x7f769012f600 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:42.105 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.099+0000 7f76949c5640 1 --2- 172.21.15.83:0/45398672 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f769012f1c0 0x7f769012f600 secure :-1 s=READY pgs=85 cs=0 l=1 rev1=1 crypto rx=0x7f7688005fd0 tx=0x7f768802a980 comp rx=0 tx=0).stop 2024-09-14T03:40:42.105 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.099+0000 7f76949c5640 1 -- 172.21.15.83:0/45398672 shutdown_connections 2024-09-14T03:40:42.105 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.099+0000 7f76949c5640 1 --2- 172.21.15.83:0/45398672 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f769012f1c0 0x7f769012f600 unknown :-1 s=CLOSED pgs=85 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:42.105 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.099+0000 7f76949c5640 1 --2- 172.21.15.83:0/45398672 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f7690098d40 0x7f7690099120 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:42.106 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.099+0000 7f76949c5640 1 -- 172.21.15.83:0/45398672 >> 172.21.15.83:0/45398672 conn(0x7f76900a1790 msgr2=0x7f76900a1ba0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:42.106 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.099+0000 7f76949c5640 1 -- 172.21.15.83:0/45398672 shutdown_connections 2024-09-14T03:40:42.106 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.099+0000 7f76949c5640 1 -- 172.21.15.83:0/45398672 wait complete. 2024-09-14T03:40:42.106 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.099+0000 7efc7a374640 1 Processor -- start 2024-09-14T03:40:42.106 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.100+0000 7efe4cb0f640 1 Processor -- start 2024-09-14T03:40:42.106 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.100+0000 7fa888803640 1 Processor -- start 2024-09-14T03:40:42.107 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.100+0000 7efc7a374640 1 -- start start 2024-09-14T03:40:42.107 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.100+0000 7efe4cb0f640 1 -- start start 2024-09-14T03:40:42.107 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.100+0000 7fa888803640 1 -- start start 2024-09-14T03:40:42.108 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.100+0000 7efc7a374640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7efc740700b0 0x7efc741282e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:42.108 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.100+0000 7efc7a374640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7efc740709d0 0x7efc74128820 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:42.108 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.100+0000 7efe4cb0f640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7efe48098d40 0x7efe481c4c70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:42.108 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.100+0000 7efc73fff640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7efc740700b0 0x7efc741282e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:42.108 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.100+0000 7efc73fff640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7efc740700b0 0x7efc741282e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3301/0 says I am v2:172.21.15.83:53142/0 (socket says 172.21.15.83:53142) 2024-09-14T03:40:42.109 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.100+0000 7efe4cb0f640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7efe48099660 0x7efe481c51b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:42.109 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.100+0000 7fa888803640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fa88412bc80 0x7fa884125850 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:42.109 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.100+0000 7efc73fff640 1 -- 172.21.15.83:0/3187697648 learned_addr learned my addr 172.21.15.83:0/3187697648 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:40:42.109 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.100+0000 7efe4659c640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7efe48098d40 0x7efe481c4c70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:42.110 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.100+0000 7efc7a374640 1 --2- 172.21.15.83:0/3187697648 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7efc7412c130 0x7efc74128d60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:42.110 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.100+0000 7efe4cb0f640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7efe4812eff0 0x7efe48071480 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:42.110 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.100+0000 7efe4659c640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7efe48098d40 0x7efe481c4c70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3302/0 says I am v2:172.21.15.83:42494/0 (socket says 172.21.15.83:42494) 2024-09-14T03:40:42.110 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.100+0000 7fa888803640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fa88412c770 0x7fa884125d90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:42.110 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.100+0000 7efc7a374640 1 -- 172.21.15.83:0/3187697648 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7efc7406f120 con 0x7efc7412c130 2024-09-14T03:40:42.111 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.100+0000 7efe4659c640 1 -- 172.21.15.83:0/1913152352 learned_addr learned my addr 172.21.15.83:0/1913152352 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:40:42.111 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.100+0000 7efc7a374640 1 -- 172.21.15.83:0/3187697648 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7efc7406efa0 con 0x7efc740700b0 2024-09-14T03:40:42.111 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.100+0000 7efe4cb0f640 1 -- 172.21.15.83:0/1913152352 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7efe48136570 con 0x7efe48099660 2024-09-14T03:40:42.111 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.100+0000 7efe4cb0f640 1 -- 172.21.15.83:0/1913152352 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7efe481363f0 con 0x7efe4812eff0 2024-09-14T03:40:42.111 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.100+0000 7efe4cb0f640 1 -- 172.21.15.83:0/1913152352 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7efe481366f0 con 0x7efe48098d40 2024-09-14T03:40:42.112 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.100+0000 7efc7a374640 1 -- 172.21.15.83:0/3187697648 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7efc7406f2a0 con 0x7efc740709d0 2024-09-14T03:40:42.112 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.100+0000 7efc737fe640 1 --2- 172.21.15.83:0/3187697648 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7efc740709d0 0x7efc74128820 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:42.112 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.101+0000 7efe46d9d640 1 --2- 172.21.15.83:0/1913152352 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7efe4812eff0 0x7efe48071480 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:42.112 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.101+0000 7efc78911640 1 --2- 172.21.15.83:0/3187697648 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7efc7412c130 0x7efc74128d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:42.113 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.101+0000 7efe45d9b640 1 --2- 172.21.15.83:0/1913152352 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7efe48099660 0x7efe481c51b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:42.113 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.101+0000 7f76949c5640 1 Processor -- start 2024-09-14T03:40:42.113 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.101+0000 7efe4659c640 1 -- 172.21.15.83:0/1913152352 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7efe4812eff0 msgr2=0x7efe48071480 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:42.113 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.101+0000 7efe4659c640 1 --2- 172.21.15.83:0/1913152352 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7efe4812eff0 0x7efe48071480 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:42.113 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.101+0000 7efe4659c640 1 -- 172.21.15.83:0/1913152352 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7efe48099660 msgr2=0x7efe481c51b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:42.113 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.101+0000 7efe4659c640 1 --2- 172.21.15.83:0/1913152352 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7efe48099660 0x7efe481c51b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:42.114 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.101+0000 7efc73fff640 1 -- 172.21.15.83:0/3187697648 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7efc740709d0 msgr2=0x7efc74128820 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:42.114 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.101+0000 7efc73fff640 1 --2- 172.21.15.83:0/3187697648 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7efc740709d0 0x7efc74128820 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:42.114 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.101+0000 7efc73fff640 1 -- 172.21.15.83:0/3187697648 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7efc7412c130 msgr2=0x7efc74128d60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:42.114 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.101+0000 7efc73fff640 1 --2- 172.21.15.83:0/3187697648 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7efc7412c130 0x7efc74128d60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:42.114 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.101+0000 7efc73fff640 1 -- 172.21.15.83:0/3187697648 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7efc74073640 con 0x7efc740700b0 2024-09-14T03:40:42.114 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.101+0000 7efe4659c640 1 -- 172.21.15.83:0/1913152352 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7efe4812ff50 con 0x7efe48098d40 2024-09-14T03:40:42.115 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.101+0000 7fa882d9d640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fa88412bc80 0x7fa884125850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:42.115 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.101+0000 7fa882d9d640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fa88412bc80 0x7fa884125850 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3302/0 says I am v2:172.21.15.83:42502/0 (socket says 172.21.15.83:42502) 2024-09-14T03:40:42.115 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.101+0000 7fa882d9d640 1 -- 172.21.15.83:0/2544210884 learned_addr learned my addr 172.21.15.83:0/2544210884 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:40:42.115 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.101+0000 7fa88259c640 1 --2- 172.21.15.83:0/2544210884 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fa88412c770 0x7fa884125d90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:42.115 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.101+0000 7fa888803640 1 --2- 172.21.15.83:0/2544210884 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fa884131470 0x7fa8841262d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:42.115 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.101+0000 7fa888803640 1 -- 172.21.15.83:0/2544210884 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7fa88406fb20 con 0x7fa88412c770 2024-09-14T03:40:42.115 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.101+0000 7fa888803640 1 -- 172.21.15.83:0/2544210884 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7fa88406f9a0 con 0x7fa884131470 2024-09-14T03:40:42.115 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.101+0000 7fa888803640 1 -- 172.21.15.83:0/2544210884 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7fa88406fca0 con 0x7fa88412bc80 2024-09-14T03:40:42.115 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.101+0000 7f76949c5640 1 -- start start 2024-09-14T03:40:42.116 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.101+0000 7fa88359e640 1 --2- 172.21.15.83:0/2544210884 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fa884131470 0x7fa8841262d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:42.116 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.101+0000 7efe4659c640 1 --2- 172.21.15.83:0/1913152352 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7efe48098d40 0x7efe481c4c70 secure :-1 s=READY pgs=55 cs=0 l=1 rev1=1 crypto rx=0x7efe38034040 tx=0x7efe38005e80 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:40:42.116 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.101+0000 7fa88259c640 1 -- 172.21.15.83:0/2544210884 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fa88412bc80 msgr2=0x7fa884125850 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:42.116 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.101+0000 7efc73fff640 1 --2- 172.21.15.83:0/3187697648 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7efc740700b0 0x7efc741282e0 secure :-1 s=READY pgs=53 cs=0 l=1 rev1=1 crypto rx=0x7efc64006010 tx=0x7efc640038a0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:40:42.117 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.101+0000 7fa88259c640 1 --2- 172.21.15.83:0/2544210884 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fa88412bc80 0x7fa884125850 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:42.117 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.101+0000 7fa88259c640 1 -- 172.21.15.83:0/2544210884 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fa884131470 msgr2=0x7fa8841262d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:42.117 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.101+0000 7fa88259c640 1 --2- 172.21.15.83:0/2544210884 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fa884131470 0x7fa8841262d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:42.117 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.101+0000 7fa88259c640 1 -- 172.21.15.83:0/2544210884 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fa884128ac0 con 0x7fa88412c770 2024-09-14T03:40:42.118 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.102+0000 7efc70ff9640 1 -- 172.21.15.83:0/3187697648 <== mon.1 v2:172.21.15.83:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7efc6403f070 con 0x7efc740700b0 2024-09-14T03:40:42.118 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.102+0000 7efe36ffd640 1 -- 172.21.15.83:0/1913152352 <== mon.2 v2:172.21.15.83:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7efe3803e030 con 0x7efe48098d40 2024-09-14T03:40:42.118 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.102+0000 7fa88259c640 1 --2- 172.21.15.83:0/2544210884 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fa88412c770 0x7fa884125d90 secure :-1 s=READY pgs=87 cs=0 l=1 rev1=1 crypto rx=0x7fa878031000 tx=0x7fa8780056f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:40:42.118 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.102+0000 7efe36ffd640 1 -- 172.21.15.83:0/1913152352 <== mon.2 v2:172.21.15.83:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7efe38003c30 con 0x7efe48098d40 2024-09-14T03:40:42.119 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.102+0000 7efe4cb0f640 1 -- 172.21.15.83:0/1913152352 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7efe480736b0 con 0x7efe48098d40 2024-09-14T03:40:42.119 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.102+0000 7efe36ffd640 1 -- 172.21.15.83:0/1913152352 <== mon.2 v2:172.21.15.83:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7efe38035040 con 0x7efe48098d40 2024-09-14T03:40:42.119 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.102+0000 7efc70ff9640 1 -- 172.21.15.83:0/3187697648 <== mon.1 v2:172.21.15.83:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7efc640047a0 con 0x7efc740700b0 2024-09-14T03:40:42.119 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.102+0000 7efc70ff9640 1 -- 172.21.15.83:0/3187697648 <== mon.1 v2:172.21.15.83:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7efc64035030 con 0x7efc740700b0 2024-09-14T03:40:42.119 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.102+0000 7efc7a374640 1 -- 172.21.15.83:0/3187697648 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7efc74096e10 con 0x7efc740700b0 2024-09-14T03:40:42.120 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.102+0000 7f76949c5640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f7690098d40 0x7f76901c4f30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:42.120 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.102+0000 7efe4cb0f640 1 -- 172.21.15.83:0/1913152352 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7efe480719c0 con 0x7efe48098d40 2024-09-14T03:40:42.121 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.102+0000 7f76949c5640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f7690099660 0x7f76901c5470 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:42.121 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.102+0000 7f768e59c640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f7690098d40 0x7f76901c4f30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:42.121 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.102+0000 7f768e59c640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f7690098d40 0x7f76901c4f30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3300/0 says I am v2:172.21.15.83:37416/0 (socket says 172.21.15.83:37416) 2024-09-14T03:40:42.121 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.102+0000 7f768dd9b640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f7690099660 0x7f76901c5470 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:42.121 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.102+0000 7f768dd9b640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f7690099660 0x7f76901c5470 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3301/0 says I am v2:172.21.15.83:53168/0 (socket says 172.21.15.83:53168) 2024-09-14T03:40:42.121 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.102+0000 7f76949c5640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f769012f1c0 0x7f7690071640 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:42.121 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.103+0000 7f76949c5640 1 -- --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7f7690137870 con 0x7f7690098d40 2024-09-14T03:40:42.121 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.103+0000 7f76949c5640 1 -- --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7f76901376f0 con 0x7f7690099660 2024-09-14T03:40:42.122 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.102+0000 7efc7a374640 1 -- 172.21.15.83:0/3187697648 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7efc7412afc0 con 0x7efc740700b0 2024-09-14T03:40:42.122 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.103+0000 7f76949c5640 1 -- --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7f76901379f0 con 0x7f769012f1c0 2024-09-14T03:40:42.122 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.102+0000 7fa86b7fe640 1 -- 172.21.15.83:0/2544210884 <== mon.0 v2:172.21.15.83:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa87803f070 con 0x7fa88412c770 2024-09-14T03:40:42.123 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.102+0000 7fa86b7fe640 1 -- 172.21.15.83:0/2544210884 <== mon.0 v2:172.21.15.83:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fa8780035f0 con 0x7fa88412c770 2024-09-14T03:40:42.123 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.102+0000 7fa86b7fe640 1 -- 172.21.15.83:0/2544210884 <== mon.0 v2:172.21.15.83:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa878035030 con 0x7fa88412c770 2024-09-14T03:40:42.123 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.103+0000 7fa888803640 1 -- 172.21.15.83:0/2544210884 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fa884127aa0 con 0x7fa88412c770 2024-09-14T03:40:42.123 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.103+0000 7fa888803640 1 -- 172.21.15.83:0/2544210884 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fa884127f10 con 0x7fa88412c770 2024-09-14T03:40:42.123 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.103+0000 7efc7a374640 1 -- 172.21.15.83:0/3187697648 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_get_version(what=osdmap handle=1) -- 0x7efc74070490 con 0x7efc740700b0 2024-09-14T03:40:42.123 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.103+0000 7efe4cb0f640 1 -- 172.21.15.83:0/1913152352 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_get_version(what=osdmap handle=1) -- 0x7efe4812eab0 con 0x7efe48098d40 2024-09-14T03:40:42.124 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.103+0000 7f768e59c640 1 -- 172.21.15.83:0/3447381539 learned_addr learned my addr 172.21.15.83:0/3447381539 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:40:42.124 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.103+0000 7f768ed9d640 1 --2- 172.21.15.83:0/3447381539 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f769012f1c0 0x7f7690071640 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:42.124 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.103+0000 7f768e59c640 1 -- 172.21.15.83:0/3447381539 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f769012f1c0 msgr2=0x7f7690071640 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:42.124 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.103+0000 7f768e59c640 1 --2- 172.21.15.83:0/3447381539 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f769012f1c0 0x7f7690071640 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:42.124 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.103+0000 7f768e59c640 1 -- 172.21.15.83:0/3447381539 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f7690099660 msgr2=0x7f76901c5470 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:42.124 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.104+0000 7f768e59c640 1 --2- 172.21.15.83:0/3447381539 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f7690099660 0x7f76901c5470 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:42.125 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.104+0000 7efe36ffd640 1 -- 172.21.15.83:0/1913152352 <== mon.2 v2:172.21.15.83:3302/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7efe38041030 con 0x7efe48098d40 2024-09-14T03:40:42.125 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.104+0000 7f768e59c640 1 -- 172.21.15.83:0/3447381539 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f7690074220 con 0x7f7690098d40 2024-09-14T03:40:42.125 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.104+0000 7f768dd9b640 1 --2- 172.21.15.83:0/3447381539 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f7690099660 0x7f76901c5470 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:40:42.125 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.104+0000 7fa888803640 1 -- 172.21.15.83:0/2544210884 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_get_version(what=osdmap handle=1) -- 0x7fa884128200 con 0x7fa88412c770 2024-09-14T03:40:42.126 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.104+0000 7f768e59c640 1 --2- 172.21.15.83:0/3447381539 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f7690098d40 0x7f76901c4f30 secure :-1 s=READY pgs=88 cs=0 l=1 rev1=1 crypto rx=0x7f768000e7c0 tx=0x7f768000ec80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:40:42.126 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.104+0000 7efe36ffd640 1 -- 172.21.15.83:0/1913152352 <== mon.2 v2:172.21.15.83:3302/0 5 ==== osd_map(33..33 src has 1..33) ==== 3027+0+0 (secure 0 0 0) 0x7efe38089570 con 0x7efe48098d40 2024-09-14T03:40:42.126 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.104+0000 7f767effd640 1 -- 172.21.15.83:0/3447381539 <== mon.0 v2:172.21.15.83:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f768000cc60 con 0x7f7690098d40 2024-09-14T03:40:42.126 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.104+0000 7efc70ff9640 1 -- 172.21.15.83:0/3187697648 <== mon.1 v2:172.21.15.83:3301/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7efc64042070 con 0x7efc740700b0 2024-09-14T03:40:42.127 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.104+0000 7efe36ffd640 1 -- 172.21.15.83:0/1913152352 --> v1:172.21.15.83:6805/543231167 -- command(tid 1: {"prefix": "get_command_descriptions"}) -- 0x7efe3803ee50 con 0x7efe28057d90 2024-09-14T03:40:42.127 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.104+0000 7efe36ffd640 1 -- 172.21.15.83:0/1913152352 <== mon.2 v2:172.21.15.83:3302/0 6 ==== mon_get_version_reply(handle=1 version=33) ==== 24+0+0 (secure 0 0 0) 0x7efe38090050 con 0x7efe48098d40 2024-09-14T03:40:42.127 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.105+0000 7fa86b7fe640 1 -- 172.21.15.83:0/2544210884 <== mon.0 v2:172.21.15.83:3300/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7fa878042070 con 0x7fa88412c770 2024-09-14T03:40:42.128 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.104+0000 7f767effd640 1 -- 172.21.15.83:0/3447381539 <== mon.0 v2:172.21.15.83:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f7680010430 con 0x7f7690098d40 2024-09-14T03:40:42.128 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.105+0000 7f767effd640 1 -- 172.21.15.83:0/3447381539 <== mon.0 v2:172.21.15.83:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7680010860 con 0x7f7690098d40 2024-09-14T03:40:42.128 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.105+0000 7f76949c5640 1 -- 172.21.15.83:0/3447381539 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f7690073b00 con 0x7f7690098d40 2024-09-14T03:40:42.128 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.105+0000 7f76949c5640 1 -- 172.21.15.83:0/3447381539 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f76901c59b0 con 0x7f7690098d40 2024-09-14T03:40:42.128 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.105+0000 7fa86b7fe640 1 -- 172.21.15.83:0/2544210884 <== mon.0 v2:172.21.15.83:3300/0 5 ==== osd_map(33..33 src has 1..33) ==== 3027+0+0 (secure 0 0 0) 0x7fa87808a580 con 0x7fa88412c770 2024-09-14T03:40:42.129 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.105+0000 7efc70ff9640 1 -- 172.21.15.83:0/3187697648 <== mon.1 v2:172.21.15.83:3301/0 5 ==== osd_map(33..33 src has 1..33) ==== 3027+0+0 (secure 0 0 0) 0x7efc6408a460 con 0x7efc740700b0 2024-09-14T03:40:42.129 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.106+0000 7efc70ff9640 1 -- 172.21.15.83:0/3187697648 --> v1:172.21.15.83:6801/3543625573 -- command(tid 1: {"prefix": "get_command_descriptions"}) -- 0x7efc64050ea0 con 0x7efc60057e70 2024-09-14T03:40:42.129 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.106+0000 7efc70ff9640 1 -- 172.21.15.83:0/3187697648 <== mon.1 v2:172.21.15.83:3301/0 6 ==== mon_get_version_reply(handle=1 version=33) ==== 24+0+0 (secure 0 0 0) 0x7efc6408a850 con 0x7efc740700b0 2024-09-14T03:40:42.129 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.106+0000 7efe36ffd640 1 -- 172.21.15.83:0/1913152352 <== osd.1 v1:172.21.15.83:6805/543231167 1 ==== command_reply(tid 1: 0 ) ==== 8+0+27193 (unknown 0 0 1302949928) 0x7efe3803ee50 con 0x7efe28057d90 2024-09-14T03:40:42.129 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.105+0000 7fa86b7fe640 1 -- 172.21.15.83:0/2544210884 --> v1:172.21.15.83:6809/905248239 -- command(tid 1: {"prefix": "get_command_descriptions"}) -- 0x7fa878036e90 con 0x7fa870057d80 2024-09-14T03:40:42.129 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.105+0000 7fa86b7fe640 1 -- 172.21.15.83:0/2544210884 <== mon.0 v2:172.21.15.83:3300/0 6 ==== mon_get_version_reply(handle=1 version=33) ==== 24+0+0 (secure 0 0 0) 0x7fa87808aa10 con 0x7fa88412c770 2024-09-14T03:40:42.130 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.106+0000 7f76949c5640 1 -- 172.21.15.83:0/3447381539 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_get_version(what=osdmap handle=1) -- 0x7f7690099120 con 0x7f7690098d40 2024-09-14T03:40:42.130 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.106+0000 7fa86b7fe640 1 -- 172.21.15.83:0/2544210884 <== osd.2 v1:172.21.15.83:6809/905248239 1 ==== command_reply(tid 1: 0 ) ==== 8+0+27193 (unknown 0 0 1302949928) 0x7fa878036e90 con 0x7fa870057d80 2024-09-14T03:40:42.130 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.106+0000 7f767effd640 1 -- 172.21.15.83:0/3447381539 <== mon.0 v2:172.21.15.83:3300/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7f7680010a00 con 0x7f7690098d40 2024-09-14T03:40:42.130 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.107+0000 7efc70ff9640 1 -- 172.21.15.83:0/3187697648 <== osd.0 v1:172.21.15.83:6801/3543625573 1 ==== command_reply(tid 1: 0 ) ==== 8+0+27193 (unknown 0 0 1302949928) 0x7efc64050ea0 con 0x7efc60057e70 2024-09-14T03:40:42.131 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.107+0000 7f767effd640 1 -- 172.21.15.83:0/3447381539 <== mon.0 v2:172.21.15.83:3300/0 5 ==== osd_map(33..33 src has 1..33) ==== 3027+0+0 (secure 0 0 0) 0x7f7680069940 con 0x7f7690098d40 2024-09-14T03:40:42.131 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.107+0000 7f767effd640 1 -- 172.21.15.83:0/3447381539 --> v1:172.21.15.83:6813/3860913217 -- command(tid 1: {"prefix": "get_command_descriptions"}) -- 0x7f767005ba50 con 0x7f7670057da0 2024-09-14T03:40:42.131 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.107+0000 7f767effd640 1 -- 172.21.15.83:0/3447381539 <== mon.0 v2:172.21.15.83:3300/0 6 ==== mon_get_version_reply(handle=1 version=33) ==== 24+0+0 (secure 0 0 0) 0x7f768006e2c0 con 0x7f7690098d40 2024-09-14T03:40:42.131 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.109+0000 7f767effd640 1 -- 172.21.15.83:0/3447381539 <== osd.3 v1:172.21.15.83:6813/3860913217 1 ==== command_reply(tid 1: 0 ) ==== 8+0+27193 (unknown 0 0 1302949928) 0x7f767005ba50 con 0x7f7670057da0 2024-09-14T03:40:42.132 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.122+0000 7efc7a374640 1 -- 172.21.15.83:0/3187697648 --> v1:172.21.15.83:6801/3543625573 -- command(tid 2: {"prefix": "flush_pg_stats"}) -- 0x7efc741295b0 con 0x7efc60057e70 2024-09-14T03:40:42.132 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.122+0000 7efc70ff9640 1 -- 172.21.15.83:0/3187697648 <== osd.0 v1:172.21.15.83:6801/3543625573 2 ==== command_reply(tid 2: 0 ) ==== 8+0+12 (unknown 0 0 3581622916) 0x7efc741295b0 con 0x7efc60057e70 2024-09-14T03:40:42.133 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.123+0000 7efc7a374640 1 -- 172.21.15.83:0/3187697648 >> v1:172.21.15.83:6801/3543625573 conn(0x7efc60057e70 legacy=0x7efc6005a2b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:42.133 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.123+0000 7efc7a374640 1 -- 172.21.15.83:0/3187697648 >> v1:172.21.15.83:6800/1803897559 conn(0x7efc60051580 legacy=0x7efc60053a40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:42.133 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.123+0000 7efc7a374640 1 -- 172.21.15.83:0/3187697648 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7efc740700b0 msgr2=0x7efc741282e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:42.133 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.123+0000 7efc7a374640 1 --2- 172.21.15.83:0/3187697648 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7efc740700b0 0x7efc741282e0 secure :-1 s=READY pgs=53 cs=0 l=1 rev1=1 crypto rx=0x7efc64006010 tx=0x7efc640038a0 comp rx=0 tx=0).stop 2024-09-14T03:40:42.133 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.123+0000 7efc78911640 1 -- 172.21.15.83:0/3187697648 reap_dead start 2024-09-14T03:40:42.133 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.123+0000 7efc7a374640 1 -- 172.21.15.83:0/3187697648 shutdown_connections 2024-09-14T03:40:42.133 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.123+0000 7efc7a374640 1 -- 172.21.15.83:0/3187697648 >> 172.21.15.83:0/3187697648 conn(0x7efc7409b9a0 msgr2=0x7efc740984a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:42.133 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.123+0000 7efc7a374640 1 -- 172.21.15.83:0/3187697648 shutdown_connections 2024-09-14T03:40:42.134 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.123+0000 7efc7a374640 1 -- 172.21.15.83:0/3187697648 wait complete. 2024-09-14T03:40:42.134 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.124+0000 7efe4cb0f640 1 -- 172.21.15.83:0/1913152352 --> v1:172.21.15.83:6805/543231167 -- command(tid 2: {"prefix": "flush_pg_stats"}) -- 0x7efe48128da0 con 0x7efe28057d90 2024-09-14T03:40:42.134 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.124+0000 7efe36ffd640 1 -- 172.21.15.83:0/1913152352 <== osd.1 v1:172.21.15.83:6805/543231167 2 ==== command_reply(tid 2: 0 ) ==== 8+0+12 (unknown 0 0 3581622916) 0x7efe48128da0 con 0x7efe28057d90 2024-09-14T03:40:42.135 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.124+0000 7efe4cb0f640 1 -- 172.21.15.83:0/1913152352 >> v1:172.21.15.83:6805/543231167 conn(0x7efe28057d90 legacy=0x7efe2805a1d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:42.135 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.124+0000 7efe4cb0f640 1 -- 172.21.15.83:0/1913152352 >> v1:172.21.15.83:6800/1803897559 conn(0x7efe280514a0 legacy=0x7efe28053960 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:42.135 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.125+0000 7efe4cb0f640 1 -- 172.21.15.83:0/1913152352 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7efe48098d40 msgr2=0x7efe481c4c70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:42.135 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.125+0000 7efe4cb0f640 1 --2- 172.21.15.83:0/1913152352 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7efe48098d40 0x7efe481c4c70 secure :-1 s=READY pgs=55 cs=0 l=1 rev1=1 crypto rx=0x7efe38034040 tx=0x7efe38005e80 comp rx=0 tx=0).stop 2024-09-14T03:40:42.135 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.125+0000 7efe46d9d640 1 -- 172.21.15.83:0/1913152352 reap_dead start 2024-09-14T03:40:42.135 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.125+0000 7efe4cb0f640 1 -- 172.21.15.83:0/1913152352 shutdown_connections 2024-09-14T03:40:42.135 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.125+0000 7efe4cb0f640 1 -- 172.21.15.83:0/1913152352 >> 172.21.15.83:0/1913152352 conn(0x7efe480a1790 msgr2=0x7efe48128440 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:42.135 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.125+0000 7efe4cb0f640 1 -- 172.21.15.83:0/1913152352 shutdown_connections 2024-09-14T03:40:42.136 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.125+0000 7efe4cb0f640 1 -- 172.21.15.83:0/1913152352 wait complete. 2024-09-14T03:40:42.136 INFO:teuthology.orchestra.run.smithi083.stdout:115964116996 2024-09-14T03:40:42.137 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.0 2024-09-14T03:40:42.137 INFO:teuthology.orchestra.run.smithi083.stdout:115964116996 2024-09-14T03:40:42.138 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.1 2024-09-14T03:40:42.140 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.139+0000 7fa888803640 1 -- 172.21.15.83:0/2544210884 --> v1:172.21.15.83:6809/905248239 -- command(tid 2: {"prefix": "flush_pg_stats"}) -- 0x7fa88409c060 con 0x7fa870057d80 2024-09-14T03:40:42.141 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.139+0000 7f76949c5640 1 -- 172.21.15.83:0/3447381539 --> v1:172.21.15.83:6813/3860913217 -- command(tid 2: {"prefix": "flush_pg_stats"}) -- 0x7f769009cb90 con 0x7f7670057da0 2024-09-14T03:40:42.145 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.140+0000 7fa86b7fe640 1 -- 172.21.15.83:0/2544210884 <== osd.2 v1:172.21.15.83:6809/905248239 2 ==== command_reply(tid 2: 0 ) ==== 8+0+12 (unknown 0 0 2085588430) 0x7fa88409c060 con 0x7fa870057d80 2024-09-14T03:40:42.145 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.140+0000 7fa888803640 1 -- 172.21.15.83:0/2544210884 >> v1:172.21.15.83:6809/905248239 conn(0x7fa870057d80 legacy=0x7fa87005a1c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:42.145 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.140+0000 7fa888803640 1 -- 172.21.15.83:0/2544210884 >> v1:172.21.15.83:6800/1803897559 conn(0x7fa870051500 legacy=0x7fa8700539c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:42.145 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.140+0000 7fa888803640 1 -- 172.21.15.83:0/2544210884 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fa88412c770 msgr2=0x7fa884125d90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:42.145 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.140+0000 7fa888803640 1 --2- 172.21.15.83:0/2544210884 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fa88412c770 0x7fa884125d90 secure :-1 s=READY pgs=87 cs=0 l=1 rev1=1 crypto rx=0x7fa878031000 tx=0x7fa8780056f0 comp rx=0 tx=0).stop 2024-09-14T03:40:42.145 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.140+0000 7fa88359e640 1 -- 172.21.15.83:0/2544210884 reap_dead start 2024-09-14T03:40:42.145 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.140+0000 7f767effd640 1 -- 172.21.15.83:0/3447381539 <== osd.3 v1:172.21.15.83:6813/3860913217 2 ==== command_reply(tid 2: 0 ) ==== 8+0+12 (unknown 0 0 1318614166) 0x7f769009cb90 con 0x7f7670057da0 2024-09-14T03:40:42.146 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.140+0000 7f76949c5640 1 -- 172.21.15.83:0/3447381539 >> v1:172.21.15.83:6813/3860913217 conn(0x7f7670057da0 legacy=0x7f767005a1e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:42.146 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.140+0000 7f76949c5640 1 -- 172.21.15.83:0/3447381539 >> v1:172.21.15.83:6800/1803897559 conn(0x7f76700514b0 legacy=0x7f7670053970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:42.146 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.140+0000 7f76949c5640 1 -- 172.21.15.83:0/3447381539 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f7690098d40 msgr2=0x7f76901c4f30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:42.146 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.140+0000 7f76949c5640 1 --2- 172.21.15.83:0/3447381539 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f7690098d40 0x7f76901c4f30 secure :-1 s=READY pgs=88 cs=0 l=1 rev1=1 crypto rx=0x7f768000e7c0 tx=0x7f768000ec80 comp rx=0 tx=0).stop 2024-09-14T03:40:42.146 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.140+0000 7f768ed9d640 1 -- 172.21.15.83:0/3447381539 reap_dead start 2024-09-14T03:40:42.146 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.140+0000 7f76949c5640 1 -- 172.21.15.83:0/3447381539 shutdown_connections 2024-09-14T03:40:42.146 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.140+0000 7f76949c5640 1 -- 172.21.15.83:0/3447381539 >> 172.21.15.83:0/3447381539 conn(0x7f76900a1790 msgr2=0x7f769009c200 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:42.146 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.140+0000 7fa888803640 1 -- 172.21.15.83:0/2544210884 shutdown_connections 2024-09-14T03:40:42.146 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.140+0000 7fa888803640 1 -- 172.21.15.83:0/2544210884 >> 172.21.15.83:0/2544210884 conn(0x7fa88409ba30 msgr2=0x7fa884132eb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:42.147 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.140+0000 7f76949c5640 1 -- 172.21.15.83:0/3447381539 shutdown_connections 2024-09-14T03:40:42.147 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.140+0000 7fa888803640 1 -- 172.21.15.83:0/2544210884 shutdown_connections 2024-09-14T03:40:42.147 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.140+0000 7f76949c5640 1 -- 172.21.15.83:0/3447381539 wait complete. 2024-09-14T03:40:42.147 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.140+0000 7fa888803640 1 -- 172.21.15.83:0/2544210884 wait complete. 2024-09-14T03:40:42.155 INFO:teuthology.orchestra.run.smithi083.stdout:124554051587 2024-09-14T03:40:42.155 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.3 2024-09-14T03:40:42.156 INFO:teuthology.orchestra.run.smithi083.stdout:128849018883 2024-09-14T03:40:42.156 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.2 2024-09-14T03:40:42.247 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.245+0000 7fb1d7ecd640 1 -- 172.21.15.83:0/3631690326 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fb1d0099830 msgr2=0x7fb1d0133dd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:42.247 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.245+0000 7fb1d7ecd640 1 --2- 172.21.15.83:0/3631690326 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fb1d0099830 0x7fb1d0133dd0 secure :-1 s=READY pgs=56 cs=0 l=1 rev1=1 crypto rx=0x7fb1c0009830 tx=0x7fb1c002a3f0 comp rx=0 tx=0).stop 2024-09-14T03:40:42.247 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.245+0000 7fb1d7ecd640 1 -- 172.21.15.83:0/3631690326 shutdown_connections 2024-09-14T03:40:42.247 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.245+0000 7fb1d7ecd640 1 --2- 172.21.15.83:0/3631690326 >> v2:172.21.15.83:3300/0 conn(0x7fb1d0134310 0x7fb1d0136700 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:42.247 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.245+0000 7fb1d7ecd640 1 --2- 172.21.15.83:0/3631690326 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fb1d0099830 0x7fb1d0133dd0 unknown :-1 s=CLOSED pgs=56 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:42.247 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.245+0000 7fb1d7ecd640 1 -- 172.21.15.83:0/3631690326 >> 172.21.15.83:0/3631690326 conn(0x7fb1d00a1790 msgr2=0x7fb1d00a1ba0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:42.248 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.245+0000 7fb1d7ecd640 1 -- 172.21.15.83:0/3631690326 shutdown_connections 2024-09-14T03:40:42.248 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.246+0000 7fb1d7ecd640 1 -- 172.21.15.83:0/3631690326 wait complete. 2024-09-14T03:40:42.249 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.246+0000 7f733510f640 1 -- 172.21.15.83:0/3910184732 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f7330099830 msgr2=0x7f7330133dd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:42.249 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.246+0000 7f733510f640 1 --2- 172.21.15.83:0/3910184732 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f7330099830 0x7f7330133dd0 secure :-1 s=READY pgs=91 cs=0 l=1 rev1=1 crypto rx=0x7f7324009830 tx=0x7f732402a400 comp rx=0 tx=0).stop 2024-09-14T03:40:42.249 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.246+0000 7f733510f640 1 -- 172.21.15.83:0/3910184732 shutdown_connections 2024-09-14T03:40:42.249 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.246+0000 7f733510f640 1 --2- 172.21.15.83:0/3910184732 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f7330134310 0x7f7330136700 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:42.249 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.246+0000 7f733510f640 1 --2- 172.21.15.83:0/3910184732 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f7330099830 0x7f7330133dd0 unknown :-1 s=CLOSED pgs=91 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:42.249 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.246+0000 7f733510f640 1 -- 172.21.15.83:0/3910184732 >> 172.21.15.83:0/3910184732 conn(0x7f73300a1790 msgr2=0x7f73300a1ba0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:42.249 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.246+0000 7f733510f640 1 -- 172.21.15.83:0/3910184732 shutdown_connections 2024-09-14T03:40:42.249 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.246+0000 7f733510f640 1 -- 172.21.15.83:0/3910184732 wait complete. 2024-09-14T03:40:42.251 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.246+0000 7fb1d7ecd640 1 Processor -- start 2024-09-14T03:40:42.251 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.246+0000 7fb1d7ecd640 1 -- start start 2024-09-14T03:40:42.251 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.247+0000 7fb1d7ecd640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fb1d0098d40 0x7fb1d01c4e20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:42.251 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.247+0000 7fb1d7ecd640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fb1d0134310 0x7fb1d01c5360 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:42.251 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.247+0000 7fb1d7ecd640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fb1d012fe40 0x7fb1d0130280 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:42.251 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.247+0000 7fb1d7ecd640 1 -- --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7fb1d0137840 con 0x7fb1d012fe40 2024-09-14T03:40:42.251 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.247+0000 7fb1d7ecd640 1 -- --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7fb1d01376c0 con 0x7fb1d0098d40 2024-09-14T03:40:42.251 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.247+0000 7fb1d7ecd640 1 -- --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7fb1d01379c0 con 0x7fb1d0134310 2024-09-14T03:40:42.251 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.247+0000 7fb1d646a640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fb1d012fe40 0x7fb1d0130280 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:42.252 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.247+0000 7fb1d5468640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fb1d0134310 0x7fb1d01c5360 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:42.252 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.247+0000 7fb1d5c69640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fb1d0098d40 0x7fb1d01c4e20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:42.252 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.247+0000 7fb1d646a640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fb1d012fe40 0x7fb1d0130280 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3300/0 says I am v2:172.21.15.83:37438/0 (socket says 172.21.15.83:37438) 2024-09-14T03:40:42.252 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.247+0000 7fb1d646a640 1 -- 172.21.15.83:0/2864232174 learned_addr learned my addr 172.21.15.83:0/2864232174 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:40:42.252 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.247+0000 7fb1d646a640 1 -- 172.21.15.83:0/2864232174 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fb1d0134310 msgr2=0x7fb1d01c5360 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:42.252 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.247+0000 7fb1d646a640 1 --2- 172.21.15.83:0/2864232174 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fb1d0134310 0x7fb1d01c5360 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:42.252 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.247+0000 7fb1d646a640 1 -- 172.21.15.83:0/2864232174 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fb1d0098d40 msgr2=0x7fb1d01c4e20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:42.252 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.248+0000 7f733510f640 1 Processor -- start 2024-09-14T03:40:42.253 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.248+0000 7fb1d646a640 1 --2- 172.21.15.83:0/2864232174 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fb1d0098d40 0x7fb1d01c4e20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:42.253 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.248+0000 7fb1d646a640 1 -- 172.21.15.83:0/2864232174 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fb1d0133400 con 0x7fb1d012fe40 2024-09-14T03:40:42.253 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.248+0000 7fb1d5468640 1 --2- 172.21.15.83:0/2864232174 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fb1d0134310 0x7fb1d01c5360 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:40:42.253 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.248+0000 7f733510f640 1 -- start start 2024-09-14T03:40:42.254 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.248+0000 7fb1d5c69640 1 --2- 172.21.15.83:0/2864232174 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fb1d0098d40 0x7fb1d01c4e20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:40:42.254 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.248+0000 7fb1d646a640 1 --2- 172.21.15.83:0/2864232174 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fb1d012fe40 0x7fb1d0130280 secure :-1 s=READY pgs=92 cs=0 l=1 rev1=1 crypto rx=0x7fb1c800c5c0 tx=0x7fb1c800ca80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:40:42.254 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.248+0000 7fb1c67fc640 1 -- 172.21.15.83:0/2864232174 <== mon.0 v2:172.21.15.83:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb1c8004d20 con 0x7fb1d012fe40 2024-09-14T03:40:42.254 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.248+0000 7fb1c67fc640 1 -- 172.21.15.83:0/2864232174 <== mon.0 v2:172.21.15.83:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fb1c8015e20 con 0x7fb1d012fe40 2024-09-14T03:40:42.254 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.248+0000 7fb1d7ecd640 1 -- 172.21.15.83:0/2864232174 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fb1d0131770 con 0x7fb1d012fe40 2024-09-14T03:40:42.254 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.248+0000 7fb1c67fc640 1 -- 172.21.15.83:0/2864232174 <== mon.0 v2:172.21.15.83:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb1c800b800 con 0x7fb1d012fe40 2024-09-14T03:40:42.255 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.248+0000 7f733510f640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f7330098d40 0x7f733012fe10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:42.255 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.248+0000 7fb1d7ecd640 1 -- 172.21.15.83:0/2864232174 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fb1d01c58a0 con 0x7fb1d012fe40 2024-09-14T03:40:42.257 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.248+0000 7f733510f640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f7330099830 0x7f7330130350 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:42.257 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.248+0000 7f733510f640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f7330134310 0x7f7330130890 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:42.257 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.248+0000 7f732e59c640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f7330099830 0x7f7330130350 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:42.258 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.248+0000 7f732ed9d640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f7330098d40 0x7f733012fe10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:42.258 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.248+0000 7f732e59c640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f7330099830 0x7f7330130350 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3301/0 says I am v2:172.21.15.83:53198/0 (socket says 172.21.15.83:53198) 2024-09-14T03:40:42.258 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.248+0000 7f732ed9d640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f7330098d40 0x7f733012fe10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3300/0 says I am v2:172.21.15.83:37440/0 (socket says 172.21.15.83:37440) 2024-09-14T03:40:42.258 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.248+0000 7f733510f640 1 -- --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7f73301378e0 con 0x7f7330098d40 2024-09-14T03:40:42.258 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.248+0000 7f733510f640 1 -- --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7f7330137760 con 0x7f7330099830 2024-09-14T03:40:42.258 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.248+0000 7f733510f640 1 -- --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7f7330137a60 con 0x7f7330134310 2024-09-14T03:40:42.258 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.249+0000 7f732e59c640 1 -- 172.21.15.83:0/424548479 learned_addr learned my addr 172.21.15.83:0/424548479 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:40:42.258 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.249+0000 7f732f59e640 1 --2- 172.21.15.83:0/424548479 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f7330134310 0x7f7330130890 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:42.258 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.249+0000 7f732ed9d640 1 -- 172.21.15.83:0/424548479 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f7330134310 msgr2=0x7f7330130890 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:42.258 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.249+0000 7f732ed9d640 1 --2- 172.21.15.83:0/424548479 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f7330134310 0x7f7330130890 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:42.258 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.249+0000 7f732ed9d640 1 -- 172.21.15.83:0/424548479 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f7330099830 msgr2=0x7f7330130350 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:42.258 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.249+0000 7f732ed9d640 1 --2- 172.21.15.83:0/424548479 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f7330099830 0x7f7330130350 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:42.259 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.249+0000 7f732ed9d640 1 -- 172.21.15.83:0/424548479 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f7330131a10 con 0x7f7330098d40 2024-09-14T03:40:42.259 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.249+0000 7f732f59e640 1 --2- 172.21.15.83:0/424548479 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f7330134310 0x7f7330130890 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:40:42.259 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.249+0000 7f732ed9d640 1 --2- 172.21.15.83:0/424548479 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f7330098d40 0x7f733012fe10 secure :-1 s=READY pgs=93 cs=0 l=1 rev1=1 crypto rx=0x7f732000c3c0 tx=0x7f732000c7c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:40:42.259 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.250+0000 7f73177fe640 1 -- 172.21.15.83:0/424548479 <== mon.0 v2:172.21.15.83:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7320017070 con 0x7f7330098d40 2024-09-14T03:40:42.259 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.250+0000 7f73177fe640 1 -- 172.21.15.83:0/424548479 <== mon.0 v2:172.21.15.83:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f73200035f0 con 0x7f7330098d40 2024-09-14T03:40:42.259 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.250+0000 7f733510f640 1 -- 172.21.15.83:0/424548479 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f7330132dc0 con 0x7f7330098d40 2024-09-14T03:40:42.259 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.250+0000 7f73177fe640 1 -- 172.21.15.83:0/424548479 <== mon.0 v2:172.21.15.83:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7320003a80 con 0x7f7330098d40 2024-09-14T03:40:42.259 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.250+0000 7fb1d7ecd640 1 -- 172.21.15.83:0/2864232174 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fb1d0136190 con 0x7fb1d012fe40 2024-09-14T03:40:42.259 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.250+0000 7f733510f640 1 -- 172.21.15.83:0/424548479 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f73301cac80 con 0x7f7330098d40 2024-09-14T03:40:42.260 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.250+0000 7fb1c67fc640 1 -- 172.21.15.83:0/2864232174 <== mon.0 v2:172.21.15.83:3300/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7fb1c800b9a0 con 0x7fb1d012fe40 2024-09-14T03:40:42.260 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.251+0000 7fb1c67fc640 1 -- 172.21.15.83:0/2864232174 <== mon.0 v2:172.21.15.83:3300/0 5 ==== osd_map(33..33 src has 1..33) ==== 3027+0+0 (secure 0 0 0) 0x7fb1c8072040 con 0x7fb1d012fe40 2024-09-14T03:40:42.260 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.251+0000 7f733510f640 1 -- 172.21.15.83:0/424548479 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f7330099120 con 0x7f7330098d40 2024-09-14T03:40:42.261 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.251+0000 7f73177fe640 1 -- 172.21.15.83:0/424548479 <== mon.0 v2:172.21.15.83:3300/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7f732001a070 con 0x7f7330098d40 2024-09-14T03:40:42.261 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.252+0000 7f73177fe640 1 -- 172.21.15.83:0/424548479 <== mon.0 v2:172.21.15.83:3300/0 5 ==== osd_map(33..33 src has 1..33) ==== 3027+0+0 (secure 0 0 0) 0x7f7320068110 con 0x7f7330098d40 2024-09-14T03:40:42.261 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.257+0000 7fb1c67fc640 1 -- 172.21.15.83:0/2864232174 <== mon.0 v2:172.21.15.83:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+157506 (secure 0 0 0) 0x7fb1d0136190 con 0x7fb1d012fe40 2024-09-14T03:40:42.265 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.264+0000 7f73177fe640 1 -- 172.21.15.83:0/424548479 <== mon.0 v2:172.21.15.83:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+157506 (secure 0 0 0) 0x7f73200671c0 con 0x7f7330098d40 2024-09-14T03:40:42.307 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.305+0000 7f2d981dc640 1 -- 172.21.15.83:0/2892466061 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f2d90098d40 msgr2=0x7f2d90099120 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:42.307 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.305+0000 7f2d981dc640 1 --2- 172.21.15.83:0/2892466061 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f2d90098d40 0x7f2d90099120 secure :-1 s=READY pgs=95 cs=0 l=1 rev1=1 crypto rx=0x7f2d8c005fd0 tx=0x7f2d8c02a4b0 comp rx=0 tx=0).stop 2024-09-14T03:40:42.308 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.306+0000 7f2d981dc640 1 -- 172.21.15.83:0/2892466061 shutdown_connections 2024-09-14T03:40:42.308 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.306+0000 7f2d981dc640 1 --2- 172.21.15.83:0/2892466061 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f2d90099660 0x7f2d9012eab0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:42.308 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.306+0000 7f2d981dc640 1 --2- 172.21.15.83:0/2892466061 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f2d90098d40 0x7f2d90099120 unknown :-1 s=CLOSED pgs=95 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:42.308 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.306+0000 7f2d981dc640 1 -- 172.21.15.83:0/2892466061 >> 172.21.15.83:0/2892466061 conn(0x7f2d900a1790 msgr2=0x7f2d900a1ba0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:42.308 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.306+0000 7f2d981dc640 1 -- 172.21.15.83:0/2892466061 shutdown_connections 2024-09-14T03:40:42.308 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.306+0000 7f2d981dc640 1 -- 172.21.15.83:0/2892466061 wait complete. 2024-09-14T03:40:42.311 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.307+0000 7f2d981dc640 1 Processor -- start 2024-09-14T03:40:42.311 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.307+0000 7f2d981dc640 1 -- start start 2024-09-14T03:40:42.312 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.308+0000 7f2d981dc640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f2d90098d40 0x7f2d90071330 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:42.312 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.308+0000 7f2d981dc640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f2d90099660 0x7f2d90071870 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:42.312 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.308+0000 7f2d981dc640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f2d9012eff0 0x7f2d90071db0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:42.312 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.308+0000 7f2d981dc640 1 -- --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7f2d90137870 con 0x7f2d90099660 2024-09-14T03:40:42.312 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.308+0000 7f2d981dc640 1 -- --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7f2d901376f0 con 0x7f2d90098d40 2024-09-14T03:40:42.312 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.308+0000 7f2d981dc640 1 -- --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7f2d901379f0 con 0x7f2d9012eff0 2024-09-14T03:40:42.312 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.308+0000 7f2d95777640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f2d90099660 0x7f2d90071870 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:42.312 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.308+0000 7f2d95777640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f2d90099660 0x7f2d90071870 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3300/0 says I am v2:172.21.15.83:39864/0 (socket says 172.21.15.83:39864) 2024-09-14T03:40:42.312 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.308+0000 7f2d95777640 1 -- 172.21.15.83:0/819542312 learned_addr learned my addr 172.21.15.83:0/819542312 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:40:42.312 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.308+0000 7f2d95777640 1 -- 172.21.15.83:0/819542312 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f2d9012eff0 msgr2=0x7f2d90071db0 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-09-14T03:40:42.312 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.308+0000 7f2d95777640 1 --2- 172.21.15.83:0/819542312 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f2d9012eff0 0x7f2d90071db0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:42.312 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.309+0000 7f2d95777640 1 -- 172.21.15.83:0/819542312 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f2d90098d40 msgr2=0x7f2d90071330 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:42.312 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.309+0000 7f2d95f78640 1 --2- 172.21.15.83:0/819542312 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f2d90098d40 0x7f2d90071330 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:42.312 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.309+0000 7f2d95777640 1 --2- 172.21.15.83:0/819542312 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f2d90098d40 0x7f2d90071330 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:42.313 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.309+0000 7f2d95777640 1 -- 172.21.15.83:0/819542312 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f2d900731c0 con 0x7f2d90099660 2024-09-14T03:40:42.313 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.309+0000 7f2d95f78640 1 --2- 172.21.15.83:0/819542312 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f2d90098d40 0x7f2d90071330 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-09-14T03:40:42.313 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.309+0000 7f2d95777640 1 --2- 172.21.15.83:0/819542312 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f2d90099660 0x7f2d90071870 secure :-1 s=READY pgs=96 cs=0 l=1 rev1=1 crypto rx=0x7f2d8000ea90 tx=0x7f2d8000ef50 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:40:42.313 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.309+0000 7f2d867fc640 1 -- 172.21.15.83:0/819542312 <== mon.0 v2:172.21.15.83:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2d8000ce40 con 0x7f2d90099660 2024-09-14T03:40:42.313 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.309+0000 7f2d981dc640 1 -- 172.21.15.83:0/819542312 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f2d900722f0 con 0x7f2d90099660 2024-09-14T03:40:42.313 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.309+0000 7f2d867fc640 1 -- 172.21.15.83:0/819542312 <== mon.0 v2:172.21.15.83:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f2d80004020 con 0x7f2d90099660 2024-09-14T03:40:42.313 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.309+0000 7f2d867fc640 1 -- 172.21.15.83:0/819542312 <== mon.0 v2:172.21.15.83:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2d80004470 con 0x7f2d90099660 2024-09-14T03:40:42.313 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.309+0000 7f2d981dc640 1 -- 172.21.15.83:0/819542312 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f2d90073da0 con 0x7f2d90099660 2024-09-14T03:40:42.314 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.311+0000 7f2d867fc640 1 -- 172.21.15.83:0/819542312 <== mon.0 v2:172.21.15.83:3300/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7f2d800035f0 con 0x7f2d90099660 2024-09-14T03:40:42.314 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.311+0000 7f2d981dc640 1 -- 172.21.15.83:0/819542312 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f2d90099120 con 0x7f2d90099660 2024-09-14T03:40:42.314 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.311+0000 7f2d867fc640 1 -- 172.21.15.83:0/819542312 <== mon.0 v2:172.21.15.83:3300/0 5 ==== osd_map(33..33 src has 1..33) ==== 3027+0+0 (secure 0 0 0) 0x7f2d80069940 con 0x7f2d90099660 2024-09-14T03:40:42.320 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.319+0000 7f2d867fc640 1 -- 172.21.15.83:0/819542312 <== mon.0 v2:172.21.15.83:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+157506 (secure 0 0 0) 0x7f2d800689f0 con 0x7f2d90099660 2024-09-14T03:40:42.322 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.319+0000 7fceff285640 1 -- 172.21.15.83:0/3507197937 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fcef8099660 msgr2=0x7fcef812eab0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:42.322 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.319+0000 7fceff285640 1 --2- 172.21.15.83:0/3507197937 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fcef8099660 0x7fcef812eab0 secure :-1 s=READY pgs=54 cs=0 l=1 rev1=1 crypto rx=0x7fcee8005fd0 tx=0x7fcee802a420 comp rx=0 tx=0).stop 2024-09-14T03:40:42.322 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.319+0000 7fceff285640 1 -- 172.21.15.83:0/3507197937 shutdown_connections 2024-09-14T03:40:42.322 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.319+0000 7fceff285640 1 --2- 172.21.15.83:0/3507197937 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fcef8099660 0x7fcef812eab0 unknown :-1 s=CLOSED pgs=54 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:42.322 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.319+0000 7fceff285640 1 --2- 172.21.15.83:0/3507197937 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fcef812eff0 0x7fcef812f480 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:42.322 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.319+0000 7fceff285640 1 --2- 172.21.15.83:0/3507197937 >> v2:172.21.15.83:3300/0 conn(0x7fcef8098d40 0x7fcef8099120 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:42.322 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.319+0000 7fceff285640 1 -- 172.21.15.83:0/3507197937 >> 172.21.15.83:0/3507197937 conn(0x7fcef80a1790 msgr2=0x7fcef80a1ba0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:42.322 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.319+0000 7fceff285640 1 -- 172.21.15.83:0/3507197937 shutdown_connections 2024-09-14T03:40:42.322 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.319+0000 7fceff285640 1 -- 172.21.15.83:0/3507197937 wait complete. 2024-09-14T03:40:42.323 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.320+0000 7fceff285640 1 Processor -- start 2024-09-14T03:40:42.323 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.320+0000 7fceff285640 1 -- start start 2024-09-14T03:40:42.325 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.321+0000 7fceff285640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fcef8098d40 0x7fcef81c4c70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:42.325 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.321+0000 7fceff285640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fcef8099660 0x7fcef81c51b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:42.325 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.321+0000 7fceff285640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fcef812eff0 0x7fcef8132a90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:42.325 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.321+0000 7fcefd021640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fcef8098d40 0x7fcef81c4c70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:42.325 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.321+0000 7fcefd021640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fcef8098d40 0x7fcef81c4c70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3302/0 says I am v2:172.21.15.83:60458/0 (socket says 172.21.15.83:60458) 2024-09-14T03:40:42.325 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.321+0000 7fcefd822640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fcef812eff0 0x7fcef8132a90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:42.325 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.321+0000 7fcefc820640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fcef8099660 0x7fcef81c51b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:42.325 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.321+0000 7fcefd822640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fcef812eff0 0x7fcef8132a90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3300/0 says I am v2:172.21.15.83:39886/0 (socket says 172.21.15.83:39886) 2024-09-14T03:40:42.325 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.321+0000 7fcefc820640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fcef8099660 0x7fcef81c51b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3301/0 says I am v2:172.21.15.83:53676/0 (socket says 172.21.15.83:53676) 2024-09-14T03:40:42.326 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.321+0000 7fcefd021640 1 -- 172.21.15.83:0/2456288084 learned_addr learned my addr 172.21.15.83:0/2456288084 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:40:42.326 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.321+0000 7fceff285640 1 -- 172.21.15.83:0/2456288084 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7fcef8136570 con 0x7fcef812eff0 2024-09-14T03:40:42.326 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.321+0000 7fceff285640 1 -- 172.21.15.83:0/2456288084 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7fcef81363f0 con 0x7fcef8099660 2024-09-14T03:40:42.326 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.321+0000 7fceff285640 1 -- 172.21.15.83:0/2456288084 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7fcef81366f0 con 0x7fcef8098d40 2024-09-14T03:40:42.326 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.322+0000 7fcefc820640 1 -- 172.21.15.83:0/2456288084 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fcef8098d40 msgr2=0x7fcef81c4c70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:42.326 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.322+0000 7fcefc820640 1 --2- 172.21.15.83:0/2456288084 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fcef8098d40 0x7fcef81c4c70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:42.326 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.322+0000 7fcefc820640 1 -- 172.21.15.83:0/2456288084 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fcef812eff0 msgr2=0x7fcef8132a90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:42.326 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.322+0000 7fcefc820640 1 --2- 172.21.15.83:0/2456288084 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fcef812eff0 0x7fcef8132a90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:42.326 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.322+0000 7fcefc820640 1 -- 172.21.15.83:0/2456288084 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fcef8074550 con 0x7fcef8099660 2024-09-14T03:40:42.326 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.322+0000 7fcefd021640 1 --2- 172.21.15.83:0/2456288084 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fcef8098d40 0x7fcef81c4c70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:40:42.326 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.322+0000 7fcefd822640 1 --2- 172.21.15.83:0/2456288084 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fcef812eff0 0x7fcef8132a90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:40:42.326 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.322+0000 7fcefc820640 1 --2- 172.21.15.83:0/2456288084 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fcef8099660 0x7fcef81c51b0 secure :-1 s=READY pgs=55 cs=0 l=1 rev1=1 crypto rx=0x7fcef812fb70 tx=0x7fcee8003ba0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:40:42.326 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.322+0000 7fceedffb640 1 -- 172.21.15.83:0/2456288084 <== mon.1 v2:172.21.15.83:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fcee803f070 con 0x7fcef8099660 2024-09-14T03:40:42.326 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.323+0000 7fceff285640 1 -- 172.21.15.83:0/2456288084 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fcef8134c50 con 0x7fcef8099660 2024-09-14T03:40:42.327 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.323+0000 7fceedffb640 1 -- 172.21.15.83:0/2456288084 <== mon.1 v2:172.21.15.83:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fcee80047a0 con 0x7fcef8099660 2024-09-14T03:40:42.327 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.323+0000 7fceedffb640 1 -- 172.21.15.83:0/2456288084 <== mon.1 v2:172.21.15.83:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fcee8035030 con 0x7fcef8099660 2024-09-14T03:40:42.327 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.323+0000 7fceff285640 1 -- 172.21.15.83:0/2456288084 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7fcef8135620 con 0x7fcef8099660 2024-09-14T03:40:42.327 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.324+0000 7fceff285640 1 -- 172.21.15.83:0/2456288084 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fcef8099120 con 0x7fcef8099660 2024-09-14T03:40:42.327 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.324+0000 7fceedffb640 1 -- 172.21.15.83:0/2456288084 <== mon.1 v2:172.21.15.83:3301/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7fcee8042070 con 0x7fcef8099660 2024-09-14T03:40:42.327 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.325+0000 7fceedffb640 1 -- 172.21.15.83:0/2456288084 <== mon.1 v2:172.21.15.83:3301/0 5 ==== osd_map(33..33 src has 1..33) ==== 3027+0+0 (secure 0 0 0) 0x7fcee8089c40 con 0x7fcef8099660 2024-09-14T03:40:42.333 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.332+0000 7fceedffb640 1 -- 172.21.15.83:0/2456288084 <== mon.1 v2:172.21.15.83:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+157506 (secure 0 0 0) 0x7fcee8088cf0 con 0x7fcef8099660 2024-09-14T03:40:42.420 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.419+0000 7fb1d7ecd640 1 -- 172.21.15.83:0/2864232174 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 0} v 0) -- 0x7fb1d0099120 con 0x7fb1d012fe40 2024-09-14T03:40:42.421 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.419+0000 7fb1c67fc640 1 -- 172.21.15.83:0/2864232174 <== mon.0 v2:172.21.15.83:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 0}]=0 v0) ==== 74+0+13 (secure 0 0 0) 0x7fb1c80710d0 con 0x7fb1d012fe40 2024-09-14T03:40:42.421 INFO:teuthology.orchestra.run.smithi083.stdout:115964116995 2024-09-14T03:40:42.423 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.421+0000 7fb1d7ecd640 1 -- 172.21.15.83:0/2864232174 >> v1:172.21.15.83:6800/1803897559 conn(0x7fb1b40514b0 legacy=0x7fb1b4053970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:42.423 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.421+0000 7fb1d7ecd640 1 -- 172.21.15.83:0/2864232174 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fb1d012fe40 msgr2=0x7fb1d0130280 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:42.423 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.421+0000 7fb1d7ecd640 1 --2- 172.21.15.83:0/2864232174 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fb1d012fe40 0x7fb1d0130280 secure :-1 s=READY pgs=92 cs=0 l=1 rev1=1 crypto rx=0x7fb1c800c5c0 tx=0x7fb1c800ca80 comp rx=0 tx=0).stop 2024-09-14T03:40:42.423 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.421+0000 7fb1d7ecd640 1 -- 172.21.15.83:0/2864232174 shutdown_connections 2024-09-14T03:40:42.423 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.421+0000 7fb1d7ecd640 1 --2- 172.21.15.83:0/2864232174 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fb1d012fe40 0x7fb1d0130280 unknown :-1 s=CLOSED pgs=92 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:42.423 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.421+0000 7fb1d7ecd640 1 --2- 172.21.15.83:0/2864232174 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fb1d0134310 0x7fb1d01c5360 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:42.423 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.421+0000 7fb1d7ecd640 1 --2- 172.21.15.83:0/2864232174 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fb1d0098d40 0x7fb1d01c4e20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:42.423 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.421+0000 7fb1d7ecd640 1 -- 172.21.15.83:0/2864232174 >> 172.21.15.83:0/2864232174 conn(0x7fb1d00a1790 msgr2=0x7fb1d0128440 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:42.423 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.421+0000 7fb1d7ecd640 1 -- 172.21.15.83:0/2864232174 shutdown_connections 2024-09-14T03:40:42.424 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.421+0000 7fb1d7ecd640 1 -- 172.21.15.83:0/2864232174 wait complete. 2024-09-14T03:40:42.433 INFO:tasks.ceph.ceph_manager.ceph:need seq 115964116996 got 115964116995 for osd.0 2024-09-14T03:40:42.461 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.459+0000 7f733510f640 1 -- 172.21.15.83:0/424548479 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 1} v 0) -- 0x7f7330131d30 con 0x7f7330098d40 2024-09-14T03:40:42.461 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.460+0000 7f73177fe640 1 -- 172.21.15.83:0/424548479 <== mon.0 v2:172.21.15.83:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 1}]=0 v0) ==== 74+0+13 (secure 0 0 0) 0x7f7320066d00 con 0x7f7330098d40 2024-09-14T03:40:42.461 INFO:teuthology.orchestra.run.smithi083.stdout:115964116995 2024-09-14T03:40:42.464 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.462+0000 7f733510f640 1 -- 172.21.15.83:0/424548479 >> v1:172.21.15.83:6800/1803897559 conn(0x7f731c0514b0 legacy=0x7f731c053970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:42.464 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.462+0000 7f733510f640 1 -- 172.21.15.83:0/424548479 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f7330098d40 msgr2=0x7f733012fe10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:42.464 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.462+0000 7f733510f640 1 --2- 172.21.15.83:0/424548479 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f7330098d40 0x7f733012fe10 secure :-1 s=READY pgs=93 cs=0 l=1 rev1=1 crypto rx=0x7f732000c3c0 tx=0x7f732000c7c0 comp rx=0 tx=0).stop 2024-09-14T03:40:42.464 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.462+0000 7f733510f640 1 -- 172.21.15.83:0/424548479 shutdown_connections 2024-09-14T03:40:42.464 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.462+0000 7f733510f640 1 --2- 172.21.15.83:0/424548479 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f7330134310 0x7f7330130890 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:42.464 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.462+0000 7f733510f640 1 --2- 172.21.15.83:0/424548479 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f7330099830 0x7f7330130350 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:42.464 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.462+0000 7f733510f640 1 --2- 172.21.15.83:0/424548479 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f7330098d40 0x7f733012fe10 unknown :-1 s=CLOSED pgs=93 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:42.464 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.462+0000 7f733510f640 1 -- 172.21.15.83:0/424548479 >> 172.21.15.83:0/424548479 conn(0x7f73300a1790 msgr2=0x7f7330128440 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:42.464 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.462+0000 7f733510f640 1 -- 172.21.15.83:0/424548479 shutdown_connections 2024-09-14T03:40:42.464 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.462+0000 7f733510f640 1 -- 172.21.15.83:0/424548479 wait complete. 2024-09-14T03:40:42.474 INFO:tasks.ceph.ceph_manager.ceph:need seq 115964116996 got 115964116995 for osd.1 2024-09-14T03:40:42.477 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.475+0000 7f2d981dc640 1 -- 172.21.15.83:0/819542312 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 3} v 0) -- 0x7f2d900698c0 con 0x7f2d90099660 2024-09-14T03:40:42.477 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.476+0000 7f2d867fc640 1 -- 172.21.15.83:0/819542312 <== mon.0 v2:172.21.15.83:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 3}]=0 v0) ==== 74+0+13 (secure 0 0 0) 0x7f2d80068530 con 0x7f2d90099660 2024-09-14T03:40:42.477 INFO:teuthology.orchestra.run.smithi083.stdout:124554051586 2024-09-14T03:40:42.479 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.478+0000 7f2d981dc640 1 -- 172.21.15.83:0/819542312 >> v1:172.21.15.83:6800/1803897559 conn(0x7f2d740514b0 legacy=0x7f2d74053970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:42.479 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.478+0000 7f2d981dc640 1 -- 172.21.15.83:0/819542312 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f2d90099660 msgr2=0x7f2d90071870 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:42.479 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.478+0000 7f2d981dc640 1 --2- 172.21.15.83:0/819542312 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f2d90099660 0x7f2d90071870 secure :-1 s=READY pgs=96 cs=0 l=1 rev1=1 crypto rx=0x7f2d8000ea90 tx=0x7f2d8000ef50 comp rx=0 tx=0).stop 2024-09-14T03:40:42.479 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.478+0000 7f2d981dc640 1 -- 172.21.15.83:0/819542312 shutdown_connections 2024-09-14T03:40:42.480 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.478+0000 7f2d981dc640 1 --2- 172.21.15.83:0/819542312 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f2d9012eff0 0x7f2d90071db0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:42.480 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.478+0000 7f2d981dc640 1 --2- 172.21.15.83:0/819542312 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f2d90099660 0x7f2d90071870 unknown :-1 s=CLOSED pgs=96 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:42.480 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.478+0000 7f2d981dc640 1 --2- 172.21.15.83:0/819542312 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f2d90098d40 0x7f2d90071330 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:42.480 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.478+0000 7f2d981dc640 1 -- 172.21.15.83:0/819542312 >> 172.21.15.83:0/819542312 conn(0x7f2d900a1790 msgr2=0x7f2d9009ca80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:42.480 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.478+0000 7f2d981dc640 1 -- 172.21.15.83:0/819542312 shutdown_connections 2024-09-14T03:40:42.480 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.478+0000 7f2d981dc640 1 -- 172.21.15.83:0/819542312 wait complete. 2024-09-14T03:40:42.490 INFO:tasks.ceph.ceph_manager.ceph:need seq 124554051587 got 124554051586 for osd.3 2024-09-14T03:40:42.501 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.500+0000 7fceff285640 1 -- 172.21.15.83:0/2456288084 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 2} v 0) -- 0x7fcef8132fd0 con 0x7fcef8099660 2024-09-14T03:40:42.502 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.500+0000 7fceedffb640 1 -- 172.21.15.83:0/2456288084 <== mon.1 v2:172.21.15.83:3301/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 2}]=0 v0) ==== 74+0+13 (secure 0 0 0) 0x7fcee8088830 con 0x7fcef8099660 2024-09-14T03:40:42.502 INFO:teuthology.orchestra.run.smithi083.stdout:128849018882 2024-09-14T03:40:42.504 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.502+0000 7fceff285640 1 -- 172.21.15.83:0/2456288084 >> v1:172.21.15.83:6800/1803897559 conn(0x7fcedc0514b0 legacy=0x7fcedc053970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:42.504 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.502+0000 7fceff285640 1 -- 172.21.15.83:0/2456288084 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fcef8099660 msgr2=0x7fcef81c51b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:42.504 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.502+0000 7fceff285640 1 --2- 172.21.15.83:0/2456288084 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fcef8099660 0x7fcef81c51b0 secure :-1 s=READY pgs=55 cs=0 l=1 rev1=1 crypto rx=0x7fcef812fb70 tx=0x7fcee8003ba0 comp rx=0 tx=0).stop 2024-09-14T03:40:42.504 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.502+0000 7fceff285640 1 -- 172.21.15.83:0/2456288084 shutdown_connections 2024-09-14T03:40:42.504 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.502+0000 7fceff285640 1 --2- 172.21.15.83:0/2456288084 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fcef812eff0 0x7fcef8132a90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:42.504 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.502+0000 7fceff285640 1 --2- 172.21.15.83:0/2456288084 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fcef8099660 0x7fcef81c51b0 unknown :-1 s=CLOSED pgs=55 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:42.505 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.502+0000 7fceff285640 1 --2- 172.21.15.83:0/2456288084 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fcef8098d40 0x7fcef81c4c70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:42.505 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.502+0000 7fceff285640 1 -- 172.21.15.83:0/2456288084 >> 172.21.15.83:0/2456288084 conn(0x7fcef80a1790 msgr2=0x7fcef8128440 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:42.505 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.502+0000 7fceff285640 1 -- 172.21.15.83:0/2456288084 shutdown_connections 2024-09-14T03:40:42.505 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:42.502+0000 7fceff285640 1 -- 172.21.15.83:0/2456288084 wait complete. 2024-09-14T03:40:42.515 INFO:tasks.ceph.ceph_manager.ceph:need seq 128849018883 got 128849018882 for osd.2 2024-09-14T03:40:43.435 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.0 2024-09-14T03:40:43.475 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.1 2024-09-14T03:40:43.491 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.3 2024-09-14T03:40:43.515 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.2 2024-09-14T03:40:43.554 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.552+0000 7f5ae7b3f640 1 -- 172.21.15.83:0/3803595510 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f5ae0097570 msgr2=0x7f5ae0097a00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:43.554 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.552+0000 7f5ae7b3f640 1 --2- 172.21.15.83:0/3803595510 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f5ae0097570 0x7f5ae0097a00 secure :-1 s=READY pgs=97 cs=0 l=1 rev1=1 crypto rx=0x7f5ad8005fd0 tx=0x7f5ad802a8f0 comp rx=0 tx=0).stop 2024-09-14T03:40:43.554 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.552+0000 7f5ae7b3f640 1 -- 172.21.15.83:0/3803595510 shutdown_connections 2024-09-14T03:40:43.554 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.552+0000 7f5ae7b3f640 1 --2- 172.21.15.83:0/3803595510 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f5ae006fad0 0x7f5ae0097030 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:43.554 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.552+0000 7f5ae7b3f640 1 --2- 172.21.15.83:0/3803595510 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f5ae0097570 0x7f5ae0097a00 unknown :-1 s=CLOSED pgs=97 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:43.555 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.552+0000 7f5ae7b3f640 1 --2- 172.21.15.83:0/3803595510 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f5ae006f1b0 0x7f5ae006f590 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:43.555 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.552+0000 7f5ae7b3f640 1 -- 172.21.15.83:0/3803595510 >> 172.21.15.83:0/3803595510 conn(0x7f5ae009bb80 msgr2=0x7f5ae009bf90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:43.555 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.553+0000 7f5ae7b3f640 1 -- 172.21.15.83:0/3803595510 shutdown_connections 2024-09-14T03:40:43.555 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.553+0000 7f5ae7b3f640 1 -- 172.21.15.83:0/3803595510 wait complete. 2024-09-14T03:40:43.557 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.553+0000 7f5ae7b3f640 1 Processor -- start 2024-09-14T03:40:43.557 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.554+0000 7f5ae7b3f640 1 -- start start 2024-09-14T03:40:43.558 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.554+0000 7f5ae7b3f640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f5ae006f1b0 0x7f5ae01c4d70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:43.558 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.554+0000 7f5ae7b3f640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f5ae006fad0 0x7f5ae01c52b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:43.558 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.554+0000 7f5ae7b3f640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f5ae0097570 0x7f5ae0092ca0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:43.558 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.554+0000 7f5ae7b3f640 1 -- --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7f5ae01346c0 con 0x7f5ae006fad0 2024-09-14T03:40:43.558 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.554+0000 7f5ae7b3f640 1 -- --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7f5ae0134540 con 0x7f5ae006f1b0 2024-09-14T03:40:43.558 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.554+0000 7f5ae7b3f640 1 -- --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7f5ae0134840 con 0x7f5ae0097570 2024-09-14T03:40:43.558 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.554+0000 7f5ae58db640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f5ae006f1b0 0x7f5ae01c4d70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:43.558 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.554+0000 7f5ae60dc640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f5ae0097570 0x7f5ae0092ca0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:43.558 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.554+0000 7f5ae58db640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f5ae006f1b0 0x7f5ae01c4d70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3301/0 says I am v2:172.21.15.83:53686/0 (socket says 172.21.15.83:53686) 2024-09-14T03:40:43.558 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.554+0000 7f5ae58db640 1 -- 172.21.15.83:0/1168004499 learned_addr learned my addr 172.21.15.83:0/1168004499 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:40:43.558 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.554+0000 7f5ae60dc640 1 --2- 172.21.15.83:0/1168004499 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f5ae0097570 0x7f5ae0092ca0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3302/0 says I am v2:172.21.15.83:60482/0 (socket says 172.21.15.83:60482) 2024-09-14T03:40:43.558 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.554+0000 7f5ae50da640 1 --2- 172.21.15.83:0/1168004499 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f5ae006fad0 0x7f5ae01c52b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:43.559 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.555+0000 7f5ae60dc640 1 -- 172.21.15.83:0/1168004499 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f5ae006f1b0 msgr2=0x7f5ae01c4d70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:43.559 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.555+0000 7f5ae60dc640 1 --2- 172.21.15.83:0/1168004499 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f5ae006f1b0 0x7f5ae01c4d70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:43.559 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.555+0000 7f5ae60dc640 1 -- 172.21.15.83:0/1168004499 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f5ae006fad0 msgr2=0x7f5ae01c52b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:43.559 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.555+0000 7f5ae60dc640 1 --2- 172.21.15.83:0/1168004499 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f5ae006fad0 0x7f5ae01c52b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:43.559 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.555+0000 7f5ae60dc640 1 -- 172.21.15.83:0/1168004499 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f5ae00738a0 con 0x7f5ae0097570 2024-09-14T03:40:43.559 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.555+0000 7f5ae58db640 1 --2- 172.21.15.83:0/1168004499 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f5ae006f1b0 0x7f5ae01c4d70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:40:43.559 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.555+0000 7f5ae50da640 1 --2- 172.21.15.83:0/1168004499 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f5ae006fad0 0x7f5ae01c52b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:40:43.559 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.555+0000 7f5ae60dc640 1 --2- 172.21.15.83:0/1168004499 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f5ae0097570 0x7f5ae0092ca0 secure :-1 s=READY pgs=57 cs=0 l=1 rev1=1 crypto rx=0x7f5ae006e920 tx=0x7f5ad8003ba0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:40:43.559 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.555+0000 7f5ad67fc640 1 -- 172.21.15.83:0/1168004499 <== mon.2 v2:172.21.15.83:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5ad8004040 con 0x7f5ae0097570 2024-09-14T03:40:43.559 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.555+0000 7f5ad67fc640 1 -- 172.21.15.83:0/1168004499 <== mon.2 v2:172.21.15.83:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f5ad802d070 con 0x7f5ae0097570 2024-09-14T03:40:43.559 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.555+0000 7f5ae7b3f640 1 -- 172.21.15.83:0/1168004499 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f5ae0096580 con 0x7f5ae0097570 2024-09-14T03:40:43.559 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.555+0000 7f5ad67fc640 1 -- 172.21.15.83:0/1168004499 <== mon.2 v2:172.21.15.83:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5ad8004460 con 0x7f5ae0097570 2024-09-14T03:40:43.559 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.555+0000 7f5ae7b3f640 1 -- 172.21.15.83:0/1168004499 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7f5ae01c57f0 con 0x7f5ae0097570 2024-09-14T03:40:43.560 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.557+0000 7f5ae7b3f640 1 -- 172.21.15.83:0/1168004499 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f5ae0097010 con 0x7f5ae0097570 2024-09-14T03:40:43.560 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.557+0000 7f5ad67fc640 1 -- 172.21.15.83:0/1168004499 <== mon.2 v2:172.21.15.83:3302/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7f5ad8004600 con 0x7f5ae0097570 2024-09-14T03:40:43.560 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.557+0000 7f5ad67fc640 1 -- 172.21.15.83:0/1168004499 <== mon.2 v2:172.21.15.83:3302/0 5 ==== osd_map(33..33 src has 1..33) ==== 3027+0+0 (secure 0 0 0) 0x7f5ad808a730 con 0x7f5ae0097570 2024-09-14T03:40:43.565 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.564+0000 7f5ad67fc640 1 -- 172.21.15.83:0/1168004499 <== mon.2 v2:172.21.15.83:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+157506 (secure 0 0 0) 0x7f5ad8048d90 con 0x7f5ae0097570 2024-09-14T03:40:43.598 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.597+0000 7fd971e3c640 1 -- 172.21.15.83:0/1488024583 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fd96c099660 msgr2=0x7fd96c12eab0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:43.599 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.597+0000 7fd971e3c640 1 --2- 172.21.15.83:0/1488024583 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fd96c099660 0x7fd96c12eab0 secure :-1 s=READY pgs=56 cs=0 l=1 rev1=1 crypto rx=0x7fd960005fd0 tx=0x7fd96002a4b0 comp rx=0 tx=0).stop 2024-09-14T03:40:43.599 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.597+0000 7fd971e3c640 1 -- 172.21.15.83:0/1488024583 shutdown_connections 2024-09-14T03:40:43.599 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.597+0000 7fd971e3c640 1 --2- 172.21.15.83:0/1488024583 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fd96c099660 0x7fd96c12eab0 unknown :-1 s=CLOSED pgs=56 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:43.599 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.597+0000 7fd971e3c640 1 --2- 172.21.15.83:0/1488024583 >> v2:172.21.15.83:3300/0 conn(0x7fd96c098d40 0x7fd96c099120 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:43.599 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.597+0000 7fd971e3c640 1 -- 172.21.15.83:0/1488024583 >> 172.21.15.83:0/1488024583 conn(0x7fd96c0a1790 msgr2=0x7fd96c0a1ba0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:43.599 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.597+0000 7fd971e3c640 1 -- 172.21.15.83:0/1488024583 shutdown_connections 2024-09-14T03:40:43.599 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.597+0000 7fd971e3c640 1 -- 172.21.15.83:0/1488024583 wait complete. 2024-09-14T03:40:43.600 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.598+0000 7fd971e3c640 1 Processor -- start 2024-09-14T03:40:43.600 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.598+0000 7fd971e3c640 1 -- start start 2024-09-14T03:40:43.600 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.598+0000 7fd971e3c640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fd96c098d40 0x7fd96c071550 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:43.600 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.599+0000 7fd971e3c640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fd96c099660 0x7fd96c071a90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:43.601 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.599+0000 7fd971e3c640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fd96c12eff0 0x7fd96c071fd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:43.601 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.599+0000 7fd971e3c640 1 -- --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7fd96c137870 con 0x7fd96c099660 2024-09-14T03:40:43.601 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.599+0000 7fd971e3c640 1 -- --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7fd96c1376f0 con 0x7fd96c098d40 2024-09-14T03:40:43.602 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.599+0000 7fd971e3c640 1 -- --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7fd96c1379f0 con 0x7fd96c12eff0 2024-09-14T03:40:43.602 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.599+0000 7fd96b7fe640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fd96c098d40 0x7fd96c071550 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:43.602 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.599+0000 7fd96b7fe640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fd96c098d40 0x7fd96c071550 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3301/0 says I am v2:172.21.15.83:53712/0 (socket says 172.21.15.83:53712) 2024-09-14T03:40:43.602 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.599+0000 7fd96b7fe640 1 -- 172.21.15.83:0/295848365 learned_addr learned my addr 172.21.15.83:0/295848365 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:40:43.602 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.599+0000 7fd96affd640 1 --2- 172.21.15.83:0/295848365 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fd96c099660 0x7fd96c071a90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:43.602 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.599+0000 7fd96bfff640 1 --2- 172.21.15.83:0/295848365 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fd96c12eff0 0x7fd96c071fd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:43.602 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.599+0000 7fd96b7fe640 1 -- 172.21.15.83:0/295848365 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fd96c12eff0 msgr2=0x7fd96c071fd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:43.602 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.599+0000 7fd96b7fe640 1 --2- 172.21.15.83:0/295848365 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fd96c12eff0 0x7fd96c071fd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:43.602 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.599+0000 7fd96b7fe640 1 -- 172.21.15.83:0/295848365 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fd96c099660 msgr2=0x7fd96c071a90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:43.602 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.599+0000 7fd96b7fe640 1 --2- 172.21.15.83:0/295848365 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fd96c099660 0x7fd96c071a90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:43.602 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.599+0000 7fd96b7fe640 1 -- 172.21.15.83:0/295848365 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fd96c072510 con 0x7fd96c098d40 2024-09-14T03:40:43.602 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.599+0000 7fd96affd640 1 --2- 172.21.15.83:0/295848365 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fd96c099660 0x7fd96c071a90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:40:43.602 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.600+0000 7fd96b7fe640 1 --2- 172.21.15.83:0/295848365 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fd96c098d40 0x7fd96c071550 secure :-1 s=READY pgs=57 cs=0 l=1 rev1=1 crypto rx=0x7fd95c00e790 tx=0x7fd95c00ec50 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:40:43.602 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.600+0000 7fd953fff640 1 -- 172.21.15.83:0/295848365 <== mon.1 v2:172.21.15.83:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd95c00eea0 con 0x7fd96c098d40 2024-09-14T03:40:43.603 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.600+0000 7fd971e3c640 1 -- 172.21.15.83:0/295848365 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fd96c073d60 con 0x7fd96c098d40 2024-09-14T03:40:43.603 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.600+0000 7fd953fff640 1 -- 172.21.15.83:0/295848365 <== mon.1 v2:172.21.15.83:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fd95c00cdb0 con 0x7fd96c098d40 2024-09-14T03:40:43.603 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.600+0000 7fd953fff640 1 -- 172.21.15.83:0/295848365 <== mon.1 v2:172.21.15.83:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd95c00eea0 con 0x7fd96c098d40 2024-09-14T03:40:43.603 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.600+0000 7fd971e3c640 1 -- 172.21.15.83:0/295848365 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7fd96c1caed0 con 0x7fd96c098d40 2024-09-14T03:40:43.603 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.601+0000 7fd971e3c640 1 -- 172.21.15.83:0/295848365 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fd96c099120 con 0x7fd96c098d40 2024-09-14T03:40:43.603 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.602+0000 7fd953fff640 1 -- 172.21.15.83:0/295848365 <== mon.1 v2:172.21.15.83:3301/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7fd95c010430 con 0x7fd96c098d40 2024-09-14T03:40:43.605 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.602+0000 7fd953fff640 1 -- 172.21.15.83:0/295848365 <== mon.1 v2:172.21.15.83:3301/0 5 ==== osd_map(33..33 src has 1..33) ==== 3027+0+0 (secure 0 0 0) 0x7fd95c00cac0 con 0x7fd96c098d40 2024-09-14T03:40:43.611 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.610+0000 7fd953fff640 1 -- 172.21.15.83:0/295848365 <== mon.1 v2:172.21.15.83:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+157506 (secure 0 0 0) 0x7fd95c045a10 con 0x7fd96c098d40 2024-09-14T03:40:43.631 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.630+0000 7f66b1508640 1 -- 172.21.15.83:0/3908088151 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f66ac12f1c0 msgr2=0x7f66ac12f600 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:43.631 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.630+0000 7f66b1508640 1 --2- 172.21.15.83:0/3908088151 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f66ac12f1c0 0x7f66ac12f600 secure :-1 s=READY pgs=58 cs=0 l=1 rev1=1 crypto rx=0x7f66a4005fd0 tx=0x7f66a402a980 comp rx=0 tx=0).stop 2024-09-14T03:40:43.632 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.630+0000 7f845cc6e640 1 -- 172.21.15.83:0/1190441991 <== mon.2 v2:172.21.15.83:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8454031600 con 0x7f8458129510 2024-09-14T03:40:43.632 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.630+0000 7f66b1508640 1 -- 172.21.15.83:0/3908088151 shutdown_connections 2024-09-14T03:40:43.632 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.630+0000 7f66b1508640 1 --2- 172.21.15.83:0/3908088151 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f66ac12f1c0 0x7f66ac12f600 unknown :-1 s=CLOSED pgs=58 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:43.633 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.630+0000 7f66b1508640 1 --2- 172.21.15.83:0/3908088151 >> v2:172.21.15.83:3300/0 conn(0x7f66ac098d40 0x7f66ac099120 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:43.633 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.630+0000 7f66b1508640 1 -- 172.21.15.83:0/3908088151 >> 172.21.15.83:0/3908088151 conn(0x7f66ac0a1790 msgr2=0x7f66ac0a1ba0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:43.633 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.630+0000 7f66b1508640 1 -- 172.21.15.83:0/3908088151 shutdown_connections 2024-09-14T03:40:43.633 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.630+0000 7f66b1508640 1 -- 172.21.15.83:0/3908088151 wait complete. 2024-09-14T03:40:43.633 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.631+0000 7f845fed4640 1 -- 172.21.15.83:0/1190441991 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f8458129510 msgr2=0x7f84581298f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:43.634 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.631+0000 7f845fed4640 1 --2- 172.21.15.83:0/1190441991 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f8458129510 0x7f84581298f0 secure :-1 s=READY pgs=59 cs=0 l=1 rev1=1 crypto rx=0x7f8454009860 tx=0x7f845402a3d0 comp rx=0 tx=0).stop 2024-09-14T03:40:43.634 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.631+0000 7f845fed4640 1 -- 172.21.15.83:0/1190441991 shutdown_connections 2024-09-14T03:40:43.634 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.631+0000 7f845fed4640 1 --2- 172.21.15.83:0/1190441991 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f8458129e30 0x7f845812c820 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:43.634 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.631+0000 7f845fed4640 1 --2- 172.21.15.83:0/1190441991 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f8458129510 0x7f84581298f0 unknown :-1 s=CLOSED pgs=59 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:43.634 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.631+0000 7f845fed4640 1 -- 172.21.15.83:0/1190441991 >> 172.21.15.83:0/1190441991 conn(0x7f845809d370 msgr2=0x7f845809d780 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:43.634 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.631+0000 7f845fed4640 1 -- 172.21.15.83:0/1190441991 shutdown_connections 2024-09-14T03:40:43.634 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.631+0000 7f845fed4640 1 -- 172.21.15.83:0/1190441991 wait complete. 2024-09-14T03:40:43.634 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.631+0000 7f66b1508640 1 Processor -- start 2024-09-14T03:40:43.635 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.632+0000 7f66b1508640 1 -- start start 2024-09-14T03:40:43.635 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.632+0000 7f845fed4640 1 Processor -- start 2024-09-14T03:40:43.635 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.632+0000 7f845fed4640 1 -- start start 2024-09-14T03:40:43.636 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.632+0000 7f66b1508640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f66ac098d40 0x7f66ac1c4f30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:43.636 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.632+0000 7f66b1508640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f66ac099660 0x7f66ac1c5470 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:43.636 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.632+0000 7f66b1508640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f66ac12f1c0 0x7f66ac0714a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:43.636 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.632+0000 7f66aa7fc640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f66ac099660 0x7f66ac1c5470 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:43.636 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.632+0000 7f66aa7fc640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f66ac099660 0x7f66ac1c5470 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3302/0 says I am v2:172.21.15.83:60512/0 (socket says 172.21.15.83:60512) 2024-09-14T03:40:43.636 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.632+0000 7f66aa7fc640 1 -- 172.21.15.83:0/2654570473 learned_addr learned my addr 172.21.15.83:0/2654570473 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:40:43.636 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.632+0000 7f66b1508640 1 -- 172.21.15.83:0/2654570473 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7f66ac137870 con 0x7f66ac098d40 2024-09-14T03:40:43.636 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.632+0000 7f66b1508640 1 -- 172.21.15.83:0/2654570473 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7f66ac1376f0 con 0x7f66ac12f1c0 2024-09-14T03:40:43.636 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.632+0000 7f66b1508640 1 -- 172.21.15.83:0/2654570473 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7f66ac1379f0 con 0x7f66ac099660 2024-09-14T03:40:43.636 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.632+0000 7f66ab7fe640 1 --2- 172.21.15.83:0/2654570473 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f66ac12f1c0 0x7f66ac0714a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:43.637 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.632+0000 7f845fed4640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f8458129510 0x7f8458130250 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:43.637 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.632+0000 7f845fed4640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f8458129e30 0x7f8458130790 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:43.637 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.632+0000 7f845fed4640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f845812cd60 0x7f8458130cd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:43.637 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.632+0000 7f845fed4640 1 -- --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7f8458135840 con 0x7f845812cd60 2024-09-14T03:40:43.637 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.632+0000 7f845fed4640 1 -- --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7f84581356c0 con 0x7f8458129510 2024-09-14T03:40:43.637 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.632+0000 7f845fed4640 1 -- --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7f84581359c0 con 0x7f8458129e30 2024-09-14T03:40:43.637 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.632+0000 7f845d46f640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f8458129e30 0x7f8458130790 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:43.638 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.632+0000 7f845d46f640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f8458129e30 0x7f8458130790 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3302/0 says I am v2:172.21.15.83:60516/0 (socket says 172.21.15.83:60516) 2024-09-14T03:40:43.638 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.632+0000 7f845d46f640 1 -- 172.21.15.83:0/3820326050 learned_addr learned my addr 172.21.15.83:0/3820326050 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:40:43.638 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.633+0000 7f845dc70640 1 --2- 172.21.15.83:0/3820326050 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f8458129510 0x7f8458130250 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:43.638 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.632+0000 7f66aaffd640 1 --2- 172.21.15.83:0/2654570473 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f66ac098d40 0x7f66ac1c4f30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:43.639 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.633+0000 7f845e471640 1 --2- 172.21.15.83:0/3820326050 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f845812cd60 0x7f8458130cd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:43.639 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.633+0000 7f845d46f640 1 -- 172.21.15.83:0/3820326050 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f8458129510 msgr2=0x7f8458130250 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:43.639 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.633+0000 7f845d46f640 1 --2- 172.21.15.83:0/3820326050 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f8458129510 0x7f8458130250 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:43.639 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.633+0000 7f845d46f640 1 -- 172.21.15.83:0/3820326050 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f845812cd60 msgr2=0x7f8458130cd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:43.639 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.633+0000 7f845d46f640 1 --2- 172.21.15.83:0/3820326050 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f845812cd60 0x7f8458130cd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:43.639 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.633+0000 7f845d46f640 1 -- 172.21.15.83:0/3820326050 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f8458126f70 con 0x7f8458129e30 2024-09-14T03:40:43.639 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.633+0000 7f66ab7fe640 1 -- 172.21.15.83:0/2654570473 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f66ac099660 msgr2=0x7f66ac1c5470 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:43.639 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.633+0000 7f66ab7fe640 1 --2- 172.21.15.83:0/2654570473 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f66ac099660 0x7f66ac1c5470 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:43.640 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.633+0000 7f66ab7fe640 1 -- 172.21.15.83:0/2654570473 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f66ac098d40 msgr2=0x7f66ac1c4f30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:43.640 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.633+0000 7f66ab7fe640 1 --2- 172.21.15.83:0/2654570473 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f66ac098d40 0x7f66ac1c4f30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:43.640 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.633+0000 7f66ab7fe640 1 -- 172.21.15.83:0/2654570473 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f66ac129460 con 0x7f66ac12f1c0 2024-09-14T03:40:43.640 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.633+0000 7f845d46f640 1 --2- 172.21.15.83:0/3820326050 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f8458129e30 0x7f8458130790 secure :-1 s=READY pgs=60 cs=0 l=1 rev1=1 crypto rx=0x7f844c00b4a0 tx=0x7f844c00b960 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:40:43.640 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.633+0000 7f66aa7fc640 1 --2- 172.21.15.83:0/2654570473 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f66ac099660 0x7f66ac1c5470 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:40:43.640 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.633+0000 7f66aaffd640 1 --2- 172.21.15.83:0/2654570473 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f66ac098d40 0x7f66ac1c4f30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-09-14T03:40:43.641 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.633+0000 7f66ab7fe640 1 --2- 172.21.15.83:0/2654570473 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f66ac12f1c0 0x7f66ac0714a0 secure :-1 s=READY pgs=58 cs=0 l=1 rev1=1 crypto rx=0x7f66a4032000 tx=0x7f66a4003f90 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:40:43.641 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.633+0000 7f844a7fc640 1 -- 172.21.15.83:0/3820326050 <== mon.2 v2:172.21.15.83:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f844c014070 con 0x7f8458129e30 2024-09-14T03:40:43.641 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.633+0000 7f844a7fc640 1 -- 172.21.15.83:0/3820326050 <== mon.2 v2:172.21.15.83:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f844c010700 con 0x7f8458129e30 2024-09-14T03:40:43.641 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.633+0000 7f845fed4640 1 -- 172.21.15.83:0/3820326050 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f8458132e40 con 0x7f8458129e30 2024-09-14T03:40:43.641 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.634+0000 7f845fed4640 1 -- 172.21.15.83:0/3820326050 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7f8458131210 con 0x7f8458129e30 2024-09-14T03:40:43.642 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.634+0000 7f844a7fc640 1 -- 172.21.15.83:0/3820326050 <== mon.2 v2:172.21.15.83:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f844c010b50 con 0x7f8458129e30 2024-09-14T03:40:43.642 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.634+0000 7f66937fe640 1 -- 172.21.15.83:0/2654570473 <== mon.1 v2:172.21.15.83:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f66a4047070 con 0x7f66ac12f1c0 2024-09-14T03:40:43.642 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.634+0000 7f66937fe640 1 -- 172.21.15.83:0/2654570473 <== mon.1 v2:172.21.15.83:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f66a40045c0 con 0x7f66ac12f1c0 2024-09-14T03:40:43.642 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.634+0000 7f66937fe640 1 -- 172.21.15.83:0/2654570473 <== mon.1 v2:172.21.15.83:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f66a4034070 con 0x7f66ac12f1c0 2024-09-14T03:40:43.642 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.635+0000 7f66b1508640 1 -- 172.21.15.83:0/2654570473 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f66ac1c59b0 con 0x7f66ac12f1c0 2024-09-14T03:40:43.642 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.635+0000 7f66b1508640 1 -- 172.21.15.83:0/2654570473 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7f66ac0740f0 con 0x7f66ac12f1c0 2024-09-14T03:40:43.643 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.635+0000 7f845fed4640 1 -- 172.21.15.83:0/3820326050 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f8440005290 con 0x7f8458129e30 2024-09-14T03:40:43.643 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.635+0000 7f844a7fc640 1 -- 172.21.15.83:0/3820326050 <== mon.2 v2:172.21.15.83:3302/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7f844c017070 con 0x7f8458129e30 2024-09-14T03:40:43.643 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.636+0000 7f844a7fc640 1 -- 172.21.15.83:0/3820326050 <== mon.2 v2:172.21.15.83:3302/0 5 ==== osd_map(33..33 src has 1..33) ==== 3027+0+0 (secure 0 0 0) 0x7f844c0689d0 con 0x7f8458129e30 2024-09-14T03:40:43.643 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.636+0000 7f66937fe640 1 -- 172.21.15.83:0/2654570473 <== mon.1 v2:172.21.15.83:3301/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7f66a404a070 con 0x7f66ac12f1c0 2024-09-14T03:40:43.644 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.637+0000 7f66937fe640 1 -- 172.21.15.83:0/2654570473 <== mon.1 v2:172.21.15.83:3301/0 5 ==== osd_map(33..33 src has 1..33) ==== 3027+0+0 (secure 0 0 0) 0x7f66a408a1c0 con 0x7f66ac12f1c0 2024-09-14T03:40:43.644 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.637+0000 7f66b1508640 1 -- 172.21.15.83:0/2654570473 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f66ac099120 con 0x7f66ac12f1c0 2024-09-14T03:40:43.644 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.639+0000 7f844a7fc640 1 -- 172.21.15.83:0/3820326050 <== mon.2 v2:172.21.15.83:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+157506 (secure 0 0 0) 0x7f844c0675a0 con 0x7f8458129e30 2024-09-14T03:40:43.644 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.641+0000 7f66937fe640 1 -- 172.21.15.83:0/2654570473 <== mon.1 v2:172.21.15.83:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+157506 (secure 0 0 0) 0x7f66a403f070 con 0x7f66ac12f1c0 2024-09-14T03:40:43.769 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.768+0000 7fd971e3c640 1 -- 172.21.15.83:0/295848365 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 1} v 0) -- 0x7fd96c072c80 con 0x7fd96c098d40 2024-09-14T03:40:43.770 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.768+0000 7fd953fff640 1 -- 172.21.15.83:0/295848365 <== mon.1 v2:172.21.15.83:3301/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 1}]=0 v0) ==== 74+0+13 (secure 0 0 0) 0x7fd95c0550d0 con 0x7fd96c098d40 2024-09-14T03:40:43.770 INFO:teuthology.orchestra.run.smithi083.stdout:115964116995 2024-09-14T03:40:43.773 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.771+0000 7fd971e3c640 1 -- 172.21.15.83:0/295848365 >> v1:172.21.15.83:6800/1803897559 conn(0x7fd9580518b0 legacy=0x7fd958053d70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:43.773 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.771+0000 7fd971e3c640 1 -- 172.21.15.83:0/295848365 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fd96c098d40 msgr2=0x7fd96c071550 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:43.773 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.771+0000 7fd971e3c640 1 --2- 172.21.15.83:0/295848365 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fd96c098d40 0x7fd96c071550 secure :-1 s=READY pgs=57 cs=0 l=1 rev1=1 crypto rx=0x7fd95c00e790 tx=0x7fd95c00ec50 comp rx=0 tx=0).stop 2024-09-14T03:40:43.773 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.771+0000 7fd971e3c640 1 -- 172.21.15.83:0/295848365 shutdown_connections 2024-09-14T03:40:43.773 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.771+0000 7fd971e3c640 1 --2- 172.21.15.83:0/295848365 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fd96c12eff0 0x7fd96c071fd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:43.773 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.771+0000 7fd971e3c640 1 --2- 172.21.15.83:0/295848365 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fd96c099660 0x7fd96c071a90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:43.773 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.771+0000 7fd971e3c640 1 --2- 172.21.15.83:0/295848365 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fd96c098d40 0x7fd96c071550 unknown :-1 s=CLOSED pgs=57 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:43.773 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.771+0000 7fd971e3c640 1 -- 172.21.15.83:0/295848365 >> 172.21.15.83:0/295848365 conn(0x7fd96c0a1790 msgr2=0x7fd96c128440 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:43.773 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.771+0000 7fd971e3c640 1 -- 172.21.15.83:0/295848365 shutdown_connections 2024-09-14T03:40:43.773 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.771+0000 7fd971e3c640 1 -- 172.21.15.83:0/295848365 wait complete. 2024-09-14T03:40:43.776 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.775+0000 7f5ae7b3f640 1 -- 172.21.15.83:0/1168004499 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 0} v 0) -- 0x7f5ae00697f0 con 0x7f5ae0097570 2024-09-14T03:40:43.777 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.776+0000 7f5ad67fc640 1 -- 172.21.15.83:0/1168004499 <== mon.2 v2:172.21.15.83:3302/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 0}]=0 v0) ==== 74+0+13 (secure 0 0 0) 0x7f5ad8089400 con 0x7f5ae0097570 2024-09-14T03:40:43.777 INFO:teuthology.orchestra.run.smithi083.stdout:115964116995 2024-09-14T03:40:43.780 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.778+0000 7f5ae7b3f640 1 -- 172.21.15.83:0/1168004499 >> v1:172.21.15.83:6800/1803897559 conn(0x7f5ac4051500 legacy=0x7f5ac40539c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:43.780 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.778+0000 7f5ae7b3f640 1 -- 172.21.15.83:0/1168004499 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f5ae0097570 msgr2=0x7f5ae0092ca0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:43.780 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.778+0000 7f5ae7b3f640 1 --2- 172.21.15.83:0/1168004499 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f5ae0097570 0x7f5ae0092ca0 secure :-1 s=READY pgs=57 cs=0 l=1 rev1=1 crypto rx=0x7f5ae006e920 tx=0x7f5ad8003ba0 comp rx=0 tx=0).stop 2024-09-14T03:40:43.780 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.778+0000 7f5ae7b3f640 1 -- 172.21.15.83:0/1168004499 shutdown_connections 2024-09-14T03:40:43.780 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.778+0000 7f5ae7b3f640 1 --2- 172.21.15.83:0/1168004499 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f5ae0097570 0x7f5ae0092ca0 unknown :-1 s=CLOSED pgs=57 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:43.780 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.778+0000 7f5ae7b3f640 1 --2- 172.21.15.83:0/1168004499 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f5ae006fad0 0x7f5ae01c52b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:43.780 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.778+0000 7f5ae7b3f640 1 --2- 172.21.15.83:0/1168004499 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f5ae006f1b0 0x7f5ae01c4d70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:43.780 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.778+0000 7f5ae7b3f640 1 -- 172.21.15.83:0/1168004499 >> 172.21.15.83:0/1168004499 conn(0x7f5ae009bb80 msgr2=0x7f5ae0099320 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:43.780 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.778+0000 7f5ae7b3f640 1 -- 172.21.15.83:0/1168004499 shutdown_connections 2024-09-14T03:40:43.780 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.778+0000 7f5ae7b3f640 1 -- 172.21.15.83:0/1168004499 wait complete. 2024-09-14T03:40:43.788 INFO:tasks.ceph.ceph_manager.ceph:need seq 115964116996 got 115964116995 for osd.1 2024-09-14T03:40:43.790 INFO:tasks.ceph.ceph_manager.ceph:need seq 115964116996 got 115964116995 for osd.0 2024-09-14T03:40:43.797 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.796+0000 7f845fed4640 1 -- 172.21.15.83:0/3820326050 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 2} v 0) -- 0x7f8440005580 con 0x7f8458129e30 2024-09-14T03:40:43.797 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.796+0000 7f844a7fc640 1 -- 172.21.15.83:0/3820326050 <== mon.2 v2:172.21.15.83:3302/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 2}]=0 v0) ==== 74+0+13 (secure 0 0 0) 0x7f844c041570 con 0x7f8458129e30 2024-09-14T03:40:43.798 INFO:teuthology.orchestra.run.smithi083.stdout:128849018882 2024-09-14T03:40:43.799 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.798+0000 7f845fed4640 1 -- 172.21.15.83:0/3820326050 >> v1:172.21.15.83:6800/1803897559 conn(0x7f843c051580 legacy=0x7f843c053a40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:43.799 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.798+0000 7f845fed4640 1 -- 172.21.15.83:0/3820326050 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f8458129e30 msgr2=0x7f8458130790 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:43.800 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.798+0000 7f845fed4640 1 --2- 172.21.15.83:0/3820326050 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f8458129e30 0x7f8458130790 secure :-1 s=READY pgs=60 cs=0 l=1 rev1=1 crypto rx=0x7f844c00b4a0 tx=0x7f844c00b960 comp rx=0 tx=0).stop 2024-09-14T03:40:43.800 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.798+0000 7f845fed4640 1 -- 172.21.15.83:0/3820326050 shutdown_connections 2024-09-14T03:40:43.800 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.798+0000 7f845fed4640 1 --2- 172.21.15.83:0/3820326050 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f845812cd60 0x7f8458130cd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:43.800 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.798+0000 7f845fed4640 1 --2- 172.21.15.83:0/3820326050 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f8458129e30 0x7f8458130790 unknown :-1 s=CLOSED pgs=60 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:43.800 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.798+0000 7f845fed4640 1 --2- 172.21.15.83:0/3820326050 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f8458129510 0x7f8458130250 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:43.800 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.798+0000 7f845fed4640 1 -- 172.21.15.83:0/3820326050 >> 172.21.15.83:0/3820326050 conn(0x7f845809d370 msgr2=0x7f8458134b40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:43.800 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.798+0000 7f845fed4640 1 -- 172.21.15.83:0/3820326050 shutdown_connections 2024-09-14T03:40:43.800 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.798+0000 7f845fed4640 1 -- 172.21.15.83:0/3820326050 wait complete. 2024-09-14T03:40:43.810 INFO:tasks.ceph.ceph_manager.ceph:need seq 128849018883 got 128849018882 for osd.2 2024-09-14T03:40:43.840 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.839+0000 7f66b1508640 1 -- 172.21.15.83:0/2654570473 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 3} v 0) -- 0x7f66ac0698c0 con 0x7f66ac12f1c0 2024-09-14T03:40:43.840 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.839+0000 7f66937fe640 1 -- 172.21.15.83:0/2654570473 <== mon.1 v2:172.21.15.83:3301/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 3}]=0 v0) ==== 74+0+13 (secure 0 0 0) 0x7f66a407c090 con 0x7f66ac12f1c0 2024-09-14T03:40:43.841 INFO:teuthology.orchestra.run.smithi083.stdout:124554051586 2024-09-14T03:40:43.843 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.841+0000 7f66b1508640 1 -- 172.21.15.83:0/2654570473 >> v1:172.21.15.83:6800/1803897559 conn(0x7f6698051500 legacy=0x7f66980539c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:43.843 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.841+0000 7f66b1508640 1 -- 172.21.15.83:0/2654570473 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f66ac12f1c0 msgr2=0x7f66ac0714a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:43.843 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.841+0000 7f66b1508640 1 --2- 172.21.15.83:0/2654570473 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f66ac12f1c0 0x7f66ac0714a0 secure :-1 s=READY pgs=58 cs=0 l=1 rev1=1 crypto rx=0x7f66a4032000 tx=0x7f66a4003f90 comp rx=0 tx=0).stop 2024-09-14T03:40:43.843 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.841+0000 7f66b1508640 1 -- 172.21.15.83:0/2654570473 shutdown_connections 2024-09-14T03:40:43.843 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.841+0000 7f66b1508640 1 --2- 172.21.15.83:0/2654570473 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f66ac12f1c0 0x7f66ac0714a0 unknown :-1 s=CLOSED pgs=58 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:43.843 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.841+0000 7f66b1508640 1 --2- 172.21.15.83:0/2654570473 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f66ac099660 0x7f66ac1c5470 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:43.844 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.841+0000 7f66b1508640 1 --2- 172.21.15.83:0/2654570473 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f66ac098d40 0x7f66ac1c4f30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:43.844 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.841+0000 7f66b1508640 1 -- 172.21.15.83:0/2654570473 >> 172.21.15.83:0/2654570473 conn(0x7f66ac0a1790 msgr2=0x7f66ac128440 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:43.844 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.841+0000 7f66b1508640 1 -- 172.21.15.83:0/2654570473 shutdown_connections 2024-09-14T03:40:43.844 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:43.841+0000 7f66b1508640 1 -- 172.21.15.83:0/2654570473 wait complete. 2024-09-14T03:40:43.853 INFO:tasks.ceph.ceph_manager.ceph:need seq 124554051587 got 124554051586 for osd.3 2024-09-14T03:40:44.789 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.1 2024-09-14T03:40:44.791 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.0 2024-09-14T03:40:44.810 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.2 2024-09-14T03:40:44.854 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.3 2024-09-14T03:40:44.898 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.896+0000 7f0419543640 1 -- 172.21.15.83:0/4150358016 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f04140712e0 msgr2=0x7f0414097740 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:44.898 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.896+0000 7f0419543640 1 --2- 172.21.15.83:0/4150358016 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f04140712e0 0x7f0414097740 secure :-1 s=READY pgs=59 cs=0 l=1 rev1=1 crypto rx=0x7f040c009ca0 tx=0x7f040c02aa70 comp rx=0 tx=0).stop 2024-09-14T03:40:44.898 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.896+0000 7f0419543640 1 -- 172.21.15.83:0/4150358016 shutdown_connections 2024-09-14T03:40:44.898 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.896+0000 7f0419543640 1 --2- 172.21.15.83:0/4150358016 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f04140712e0 0x7f0414097740 unknown :-1 s=CLOSED pgs=59 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:44.898 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.896+0000 7f0419543640 1 --2- 172.21.15.83:0/4150358016 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f04140705e0 0x7f04140709c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:44.898 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.896+0000 7f0419543640 1 -- 172.21.15.83:0/4150358016 >> 172.21.15.83:0/4150358016 conn(0x7f041409dde0 msgr2=0x7f041409e1f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:44.898 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.896+0000 7f0419543640 1 -- 172.21.15.83:0/4150358016 shutdown_connections 2024-09-14T03:40:44.898 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.896+0000 7f0419543640 1 -- 172.21.15.83:0/4150358016 wait complete. 2024-09-14T03:40:44.899 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.896+0000 7fbbcfe9a640 1 -- 172.21.15.83:0/2061516743 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fbbc81322e0 msgr2=0x7fbbc81346d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:44.899 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.896+0000 7fbbcfe9a640 1 --2- 172.21.15.83:0/2061516743 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fbbc81322e0 0x7fbbc81346d0 secure :-1 s=READY pgs=103 cs=0 l=1 rev1=1 crypto rx=0x7fbbc0005fd0 tx=0x7fbbc002a8e0 comp rx=0 tx=0).stop 2024-09-14T03:40:44.899 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.896+0000 7fbbcfe9a640 1 -- 172.21.15.83:0/2061516743 shutdown_connections 2024-09-14T03:40:44.899 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.896+0000 7fbbcfe9a640 1 --2- 172.21.15.83:0/2061516743 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fbbc81322e0 0x7fbbc81346d0 unknown :-1 s=CLOSED pgs=103 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:44.899 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.896+0000 7fbbcfe9a640 1 --2- 172.21.15.83:0/2061516743 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fbbc806edc0 0x7fbbc8131da0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:44.900 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.896+0000 7fbbcfe9a640 1 -- 172.21.15.83:0/2061516743 >> 172.21.15.83:0/2061516743 conn(0x7fbbc809e490 msgr2=0x7fbbc809e8a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:44.900 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.896+0000 7fbbcfe9a640 1 -- 172.21.15.83:0/2061516743 shutdown_connections 2024-09-14T03:40:44.900 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.896+0000 7fbbcfe9a640 1 -- 172.21.15.83:0/2061516743 wait complete. 2024-09-14T03:40:44.900 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.897+0000 7f0419543640 1 Processor -- start 2024-09-14T03:40:44.900 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.897+0000 7f0419543640 1 -- start start 2024-09-14T03:40:44.900 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.897+0000 7f0419543640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f04140705e0 0x7f0414125c10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:44.901 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.897+0000 7f0419543640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f0414070f00 0x7f0414126150 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:44.901 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.898+0000 7fbbcfe9a640 1 Processor -- start 2024-09-14T03:40:44.901 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.898+0000 7fbbcfe9a640 1 -- start start 2024-09-14T03:40:44.902 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.897+0000 7f0419543640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f04140712e0 0x7f0414126690 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:44.902 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.897+0000 7f0419543640 1 -- --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7f041406f9d0 con 0x7f04140712e0 2024-09-14T03:40:44.902 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.897+0000 7f0419543640 1 -- --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7f041406f850 con 0x7f04140705e0 2024-09-14T03:40:44.902 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.897+0000 7f0419543640 1 -- --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7f041406fb50 con 0x7f0414070f00 2024-09-14T03:40:44.902 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.897+0000 7f04137fe640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f04140712e0 0x7f0414126690 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:44.902 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.897+0000 7f04137fe640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f04140712e0 0x7f0414126690 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3300/0 says I am v2:172.21.15.83:39988/0 (socket says 172.21.15.83:39988) 2024-09-14T03:40:44.902 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.897+0000 7f04137fe640 1 -- 172.21.15.83:0/3988886264 learned_addr learned my addr 172.21.15.83:0/3988886264 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:40:44.903 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.898+0000 7f0412ffd640 1 --2- 172.21.15.83:0/3988886264 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f04140705e0 0x7f0414125c10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:44.903 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.897+0000 7f040bfff640 1 --2- 172.21.15.83:0/3988886264 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f0414070f00 0x7f0414126150 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:44.903 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.898+0000 7f04137fe640 1 -- 172.21.15.83:0/3988886264 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f0414070f00 msgr2=0x7f0414126150 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:44.903 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.898+0000 7f04137fe640 1 --2- 172.21.15.83:0/3988886264 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f0414070f00 0x7f0414126150 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:44.903 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.898+0000 7f04137fe640 1 -- 172.21.15.83:0/3988886264 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f04140705e0 msgr2=0x7f0414125c10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:44.903 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.898+0000 7f04137fe640 1 --2- 172.21.15.83:0/3988886264 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f04140705e0 0x7f0414125c10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:44.903 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.898+0000 7f04137fe640 1 -- 172.21.15.83:0/3988886264 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f0414127560 con 0x7f04140712e0 2024-09-14T03:40:44.903 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.898+0000 7f0412ffd640 1 --2- 172.21.15.83:0/3988886264 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f04140705e0 0x7f0414125c10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:40:44.903 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.898+0000 7f040bfff640 1 --2- 172.21.15.83:0/3988886264 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f0414070f00 0x7f0414126150 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:40:44.903 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.898+0000 7f04137fe640 1 --2- 172.21.15.83:0/3988886264 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f04140712e0 0x7f0414126690 secure :-1 s=READY pgs=104 cs=0 l=1 rev1=1 crypto rx=0x7f040c009bf0 tx=0x7f040c003ba0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:40:44.903 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.898+0000 7f040b7fe640 1 -- 172.21.15.83:0/3988886264 <== mon.0 v2:172.21.15.83:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f040c03f580 con 0x7f04140712e0 2024-09-14T03:40:44.905 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.898+0000 7fbbcfe9a640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fbbc806e2d0 0x7fbbc81c4a90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:44.905 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.898+0000 7fbbcfe9a640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fbbc806edc0 0x7fbbc81c4fd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:44.905 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.899+0000 7fbbcfe9a640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fbbc81322e0 0x7fbbc81c5510 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:44.905 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.899+0000 7fbbcfe9a640 1 -- --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7fbbc8135690 con 0x7fbbc806e2d0 2024-09-14T03:40:44.905 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.899+0000 7fbbcfe9a640 1 -- --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7fbbc8135510 con 0x7fbbc806edc0 2024-09-14T03:40:44.905 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.899+0000 7fbbcfe9a640 1 -- --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7fbbc8135810 con 0x7fbbc81322e0 2024-09-14T03:40:44.905 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.899+0000 7fbbcd435640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fbbc806edc0 0x7fbbc81c4fd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:44.905 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.899+0000 7fbbcd435640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fbbc806edc0 0x7fbbc81c4fd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3301/0 says I am v2:172.21.15.83:53782/0 (socket says 172.21.15.83:53782) 2024-09-14T03:40:44.905 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.899+0000 7fbbcd435640 1 -- 172.21.15.83:0/2730178816 learned_addr learned my addr 172.21.15.83:0/2730178816 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:40:44.906 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.899+0000 7fbbcdc36640 1 --2- 172.21.15.83:0/2730178816 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fbbc806e2d0 0x7fbbc81c4a90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:44.906 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.899+0000 7fbbce437640 1 --2- 172.21.15.83:0/2730178816 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fbbc81322e0 0x7fbbc81c5510 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:44.906 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.899+0000 7fbbcd435640 1 -- 172.21.15.83:0/2730178816 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fbbc81322e0 msgr2=0x7fbbc81c5510 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:44.906 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.899+0000 7fbbcd435640 1 --2- 172.21.15.83:0/2730178816 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fbbc81322e0 0x7fbbc81c5510 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:44.906 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.899+0000 7fbbcd435640 1 -- 172.21.15.83:0/2730178816 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fbbc806e2d0 msgr2=0x7fbbc81c4a90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:44.906 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.899+0000 7fbbcd435640 1 --2- 172.21.15.83:0/2730178816 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fbbc806e2d0 0x7fbbc81c4a90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:44.906 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.899+0000 7fbbcd435640 1 -- 172.21.15.83:0/2730178816 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fbbc8073d90 con 0x7fbbc806edc0 2024-09-14T03:40:44.906 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.898+0000 7f040b7fe640 1 -- 172.21.15.83:0/3988886264 <== mon.0 v2:172.21.15.83:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f040c03f720 con 0x7f04140712e0 2024-09-14T03:40:44.906 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.898+0000 7f040b7fe640 1 -- 172.21.15.83:0/3988886264 <== mon.0 v2:172.21.15.83:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f040c004040 con 0x7f04140712e0 2024-09-14T03:40:44.906 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.898+0000 7f0419543640 1 -- 172.21.15.83:0/3988886264 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f04140976b0 con 0x7f04140712e0 2024-09-14T03:40:44.907 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.899+0000 7fbbce437640 1 --2- 172.21.15.83:0/2730178816 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fbbc81322e0 0x7fbbc81c5510 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-09-14T03:40:44.907 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.899+0000 7fbbcd435640 1 --2- 172.21.15.83:0/2730178816 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fbbc806edc0 0x7fbbc81c4fd0 secure :-1 s=READY pgs=60 cs=0 l=1 rev1=1 crypto rx=0x7fbbb800e9f0 tx=0x7fbbb800eeb0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:40:44.907 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.899+0000 7fbbbe7fc640 1 -- 172.21.15.83:0/2730178816 <== mon.1 v2:172.21.15.83:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fbbb800cd80 con 0x7fbbc806edc0 2024-09-14T03:40:44.907 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.899+0000 7fbbbe7fc640 1 -- 172.21.15.83:0/2730178816 <== mon.1 v2:172.21.15.83:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fbbb8010430 con 0x7fbbc806edc0 2024-09-14T03:40:44.907 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.899+0000 7fbbbe7fc640 1 -- 172.21.15.83:0/2730178816 <== mon.1 v2:172.21.15.83:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fbbb8010840 con 0x7fbbc806edc0 2024-09-14T03:40:44.908 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.900+0000 7f0419543640 1 -- 172.21.15.83:0/3988886264 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f0414128880 con 0x7f04140712e0 2024-09-14T03:40:44.908 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.900+0000 7fbbcfe9a640 1 -- 172.21.15.83:0/2730178816 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fbbc8124100 con 0x7fbbc806edc0 2024-09-14T03:40:44.908 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.900+0000 7fbbcfe9a640 1 -- 172.21.15.83:0/2730178816 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7fbbc8092580 con 0x7fbbc806edc0 2024-09-14T03:40:44.908 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.901+0000 7f040b7fe640 1 -- 172.21.15.83:0/3988886264 <== mon.0 v2:172.21.15.83:3300/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7f040c0041e0 con 0x7f04140712e0 2024-09-14T03:40:44.909 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.901+0000 7fbbcfe9a640 1 -- 172.21.15.83:0/2730178816 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fbbc806e6b0 con 0x7fbbc806edc0 2024-09-14T03:40:44.909 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.901+0000 7f0419543640 1 -- 172.21.15.83:0/3988886264 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f0414127d90 con 0x7f04140712e0 2024-09-14T03:40:44.909 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.901+0000 7f040b7fe640 1 -- 172.21.15.83:0/3988886264 <== mon.0 v2:172.21.15.83:3300/0 5 ==== osd_map(33..33 src has 1..33) ==== 3027+0+0 (secure 0 0 0) 0x7f040c08a3b0 con 0x7f04140712e0 2024-09-14T03:40:44.909 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.901+0000 7fbbbe7fc640 1 -- 172.21.15.83:0/2730178816 <== mon.1 v2:172.21.15.83:3301/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7fbbb80109e0 con 0x7fbbc806edc0 2024-09-14T03:40:44.910 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.902+0000 7fbbbe7fc640 1 -- 172.21.15.83:0/2730178816 <== mon.1 v2:172.21.15.83:3301/0 5 ==== osd_map(33..33 src has 1..33) ==== 3027+0+0 (secure 0 0 0) 0x7fbbb8069ca0 con 0x7fbbc806edc0 2024-09-14T03:40:44.911 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.910+0000 7fbbbe7fc640 1 -- 172.21.15.83:0/2730178816 <== mon.1 v2:172.21.15.83:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+157506 (secure 0 0 0) 0x7fbbb8068c30 con 0x7fbbc806edc0 2024-09-14T03:40:44.911 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.910+0000 7f040b7fe640 1 -- 172.21.15.83:0/3988886264 <== mon.0 v2:172.21.15.83:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+157506 (secure 0 0 0) 0x7f040c04e020 con 0x7f04140712e0 2024-09-14T03:40:44.943 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.941+0000 7f4aae41d640 1 -- 172.21.15.83:0/1582358757 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f4aa8099660 msgr2=0x7f4aa812eab0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:44.943 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.941+0000 7f4aae41d640 1 --2- 172.21.15.83:0/1582358757 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f4aa8099660 0x7f4aa812eab0 secure :-1 s=READY pgs=106 cs=0 l=1 rev1=1 crypto rx=0x7f4a9c009830 tx=0x7f4a9c02a3f0 comp rx=0 tx=0).stop 2024-09-14T03:40:44.943 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.941+0000 7f4aae41d640 1 -- 172.21.15.83:0/1582358757 shutdown_connections 2024-09-14T03:40:44.943 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.941+0000 7f4aae41d640 1 --2- 172.21.15.83:0/1582358757 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f4aa8099660 0x7f4aa812eab0 unknown :-1 s=CLOSED pgs=106 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:44.943 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.941+0000 7f4aae41d640 1 --2- 172.21.15.83:0/1582358757 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f4aa8098d40 0x7f4aa8099120 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:44.944 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.941+0000 7f4aae41d640 1 -- 172.21.15.83:0/1582358757 >> 172.21.15.83:0/1582358757 conn(0x7f4aa80a1790 msgr2=0x7f4aa80a1ba0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:44.944 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.941+0000 7f4aae41d640 1 -- 172.21.15.83:0/1582358757 shutdown_connections 2024-09-14T03:40:44.944 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.941+0000 7f4aae41d640 1 -- 172.21.15.83:0/1582358757 wait complete. 2024-09-14T03:40:44.946 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.942+0000 7f4aae41d640 1 Processor -- start 2024-09-14T03:40:44.946 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.942+0000 7f4aae41d640 1 -- start start 2024-09-14T03:40:44.946 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.943+0000 7f4aae41d640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f4aa8098d40 0x7f4aa8071800 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:44.946 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.943+0000 7f4aae41d640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f4aa8099660 0x7f4aa8071d40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:44.946 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.943+0000 7f4aae41d640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f4aa812eff0 0x7f4aa8072280 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:44.946 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.943+0000 7f4aa77fe640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f4aa8099660 0x7f4aa8071d40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:44.946 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.943+0000 7f4aae41d640 1 -- --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7f4aa8137870 con 0x7f4aa812eff0 2024-09-14T03:40:44.946 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.943+0000 7f4aae41d640 1 -- --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7f4aa81376f0 con 0x7f4aa8098d40 2024-09-14T03:40:44.947 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.943+0000 7f4aae41d640 1 -- --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7f4aa81379f0 con 0x7f4aa8099660 2024-09-14T03:40:44.947 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.943+0000 7f4aa7fff640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f4aa8098d40 0x7f4aa8071800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:44.947 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.943+0000 7f4aa77fe640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f4aa8099660 0x7f4aa8071d40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3302/0 says I am v2:172.21.15.83:60554/0 (socket says 172.21.15.83:60554) 2024-09-14T03:40:44.947 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.943+0000 7f4aa77fe640 1 -- 172.21.15.83:0/2361069685 learned_addr learned my addr 172.21.15.83:0/2361069685 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:40:44.947 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.943+0000 7f4aac9ba640 1 --2- 172.21.15.83:0/2361069685 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f4aa812eff0 0x7f4aa8072280 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:44.947 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.943+0000 7f4aa77fe640 1 -- 172.21.15.83:0/2361069685 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f4aa8098d40 msgr2=0x7f4aa8071800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:44.947 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.943+0000 7f4aa77fe640 1 --2- 172.21.15.83:0/2361069685 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f4aa8098d40 0x7f4aa8071800 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:44.948 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.943+0000 7f4aa77fe640 1 -- 172.21.15.83:0/2361069685 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f4aa812eff0 msgr2=0x7f4aa8072280 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:44.948 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.943+0000 7f4aa77fe640 1 --2- 172.21.15.83:0/2361069685 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f4aa812eff0 0x7f4aa8072280 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:44.948 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.943+0000 7f4aa77fe640 1 -- 172.21.15.83:0/2361069685 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f4aa8129490 con 0x7f4aa8099660 2024-09-14T03:40:44.948 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.944+0000 7f4aa77fe640 1 --2- 172.21.15.83:0/2361069685 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f4aa8099660 0x7f4aa8071d40 secure :-1 s=READY pgs=61 cs=0 l=1 rev1=1 crypto rx=0x7f4a9c009950 tx=0x7f4a9c032cd0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:40:44.948 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.944+0000 7f4aa4ff9640 1 -- 172.21.15.83:0/2361069685 <== mon.2 v2:172.21.15.83:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4a9c040070 con 0x7f4aa8099660 2024-09-14T03:40:44.948 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.944+0000 7f4aa4ff9640 1 -- 172.21.15.83:0/2361069685 <== mon.2 v2:172.21.15.83:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f4a9c005310 con 0x7f4aa8099660 2024-09-14T03:40:44.948 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.944+0000 7f4aa4ff9640 1 -- 172.21.15.83:0/2361069685 <== mon.2 v2:172.21.15.83:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4a9c003800 con 0x7f4aa8099660 2024-09-14T03:40:44.948 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.944+0000 7f4aae41d640 1 -- 172.21.15.83:0/2361069685 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f4aa80727c0 con 0x7f4aa8099660 2024-09-14T03:40:44.948 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.944+0000 7f4aae41d640 1 -- 172.21.15.83:0/2361069685 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7f4aa80741e0 con 0x7f4aa8099660 2024-09-14T03:40:44.949 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.946+0000 7f4aa4ff9640 1 -- 172.21.15.83:0/2361069685 <== mon.2 v2:172.21.15.83:3302/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7f4a9c039040 con 0x7f4aa8099660 2024-09-14T03:40:44.949 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.946+0000 7f4aae41d640 1 -- 172.21.15.83:0/2361069685 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f4aa8099120 con 0x7f4aa8099660 2024-09-14T03:40:44.949 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.946+0000 7f4aa4ff9640 1 -- 172.21.15.83:0/2361069685 <== mon.2 v2:172.21.15.83:3302/0 5 ==== osd_map(33..33 src has 1..33) ==== 3027+0+0 (secure 0 0 0) 0x7f4a9c08a250 con 0x7f4aa8099660 2024-09-14T03:40:44.954 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.953+0000 7f4aa4ff9640 1 -- 172.21.15.83:0/2361069685 <== mon.2 v2:172.21.15.83:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+157506 (secure 0 0 0) 0x7f4aa8129490 con 0x7f4aa8099660 2024-09-14T03:40:44.983 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.982+0000 7fc05f83e640 1 -- 172.21.15.83:0/560627187 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fc058099660 msgr2=0x7fc05812eab0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:44.983 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.982+0000 7fc05f83e640 1 --2- 172.21.15.83:0/560627187 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fc058099660 0x7fc05812eab0 secure :-1 s=READY pgs=108 cs=0 l=1 rev1=1 crypto rx=0x7fc048009830 tx=0x7fc04802a3f0 comp rx=0 tx=0).stop 2024-09-14T03:40:44.984 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.982+0000 7fc05f83e640 1 -- 172.21.15.83:0/560627187 shutdown_connections 2024-09-14T03:40:44.984 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.982+0000 7fc05f83e640 1 --2- 172.21.15.83:0/560627187 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fc058099660 0x7fc05812eab0 unknown :-1 s=CLOSED pgs=108 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:44.984 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.982+0000 7fc05f83e640 1 --2- 172.21.15.83:0/560627187 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fc058098d40 0x7fc058099120 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:44.984 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.982+0000 7fc05f83e640 1 -- 172.21.15.83:0/560627187 >> 172.21.15.83:0/560627187 conn(0x7fc0580a1790 msgr2=0x7fc0580a1ba0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:44.984 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.982+0000 7fc05f83e640 1 -- 172.21.15.83:0/560627187 shutdown_connections 2024-09-14T03:40:44.984 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.982+0000 7fc05f83e640 1 -- 172.21.15.83:0/560627187 wait complete. 2024-09-14T03:40:44.986 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.983+0000 7fc05f83e640 1 Processor -- start 2024-09-14T03:40:44.986 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.983+0000 7fc05f83e640 1 -- start start 2024-09-14T03:40:44.986 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.983+0000 7fc05f83e640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fc058098d40 0x7fc0581c4d50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:44.986 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.983+0000 7fc05f83e640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fc058099660 0x7fc0581c5290 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:44.986 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.983+0000 7fc05f83e640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fc05812eff0 0x7fc058071570 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:44.986 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.983+0000 7fc05f83e640 1 -- --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7fc058137870 con 0x7fc05812eff0 2024-09-14T03:40:44.987 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.983+0000 7fc05f83e640 1 -- --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7fc0581376f0 con 0x7fc058098d40 2024-09-14T03:40:44.987 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.983+0000 7fc05f83e640 1 -- --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7fc0581379f0 con 0x7fc058099660 2024-09-14T03:40:44.987 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.983+0000 7fc05cdd9640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fc058099660 0x7fc0581c5290 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:44.987 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.983+0000 7fc05cdd9640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fc058099660 0x7fc0581c5290 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3302/0 says I am v2:172.21.15.83:60562/0 (socket says 172.21.15.83:60562) 2024-09-14T03:40:44.987 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.984+0000 7fc05cdd9640 1 -- 172.21.15.83:0/948244945 learned_addr learned my addr 172.21.15.83:0/948244945 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:40:44.987 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.983+0000 7fc05dddb640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fc05812eff0 0x7fc058071570 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:44.987 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.984+0000 7fc05d5da640 1 --2- 172.21.15.83:0/948244945 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fc058098d40 0x7fc0581c4d50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:44.987 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.984+0000 7fc05d5da640 1 -- 172.21.15.83:0/948244945 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fc058099660 msgr2=0x7fc0581c5290 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:44.987 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.984+0000 7fc05d5da640 1 --2- 172.21.15.83:0/948244945 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fc058099660 0x7fc0581c5290 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:44.987 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.984+0000 7fc05d5da640 1 -- 172.21.15.83:0/948244945 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fc05812eff0 msgr2=0x7fc058071570 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:44.987 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.984+0000 7fc05d5da640 1 --2- 172.21.15.83:0/948244945 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fc05812eff0 0x7fc058071570 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:44.987 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.984+0000 7fc05d5da640 1 -- 172.21.15.83:0/948244945 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fc058129460 con 0x7fc058098d40 2024-09-14T03:40:44.987 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.984+0000 7fc05dddb640 1 --2- 172.21.15.83:0/948244945 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fc05812eff0 0x7fc058071570 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:40:44.987 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.984+0000 7fc05cdd9640 1 --2- 172.21.15.83:0/948244945 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fc058099660 0x7fc0581c5290 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:40:44.988 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.984+0000 7fc05d5da640 1 --2- 172.21.15.83:0/948244945 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fc058098d40 0x7fc0581c4d50 secure :-1 s=READY pgs=61 cs=0 l=1 rev1=1 crypto rx=0x7fc054005fd0 tx=0x7fc05400ee50 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:40:44.988 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.984+0000 7fc04dffb640 1 -- 172.21.15.83:0/948244945 <== mon.1 v2:172.21.15.83:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc05400cc40 con 0x7fc058098d40 2024-09-14T03:40:44.988 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.984+0000 7fc04dffb640 1 -- 172.21.15.83:0/948244945 <== mon.1 v2:172.21.15.83:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fc05400cde0 con 0x7fc058098d40 2024-09-14T03:40:44.988 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.984+0000 7fc05f83e640 1 -- 172.21.15.83:0/948244945 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fc058073710 con 0x7fc058098d40 2024-09-14T03:40:44.988 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.985+0000 7fc04dffb640 1 -- 172.21.15.83:0/948244945 <== mon.1 v2:172.21.15.83:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc054004260 con 0x7fc058098d40 2024-09-14T03:40:44.988 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.985+0000 7fc05f83e640 1 -- 172.21.15.83:0/948244945 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7fc0581c59b0 con 0x7fc058098d40 2024-09-14T03:40:44.988 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.986+0000 7fc05f83e640 1 -- 172.21.15.83:0/948244945 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fc058099120 con 0x7fc058098d40 2024-09-14T03:40:44.989 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.986+0000 7fc04dffb640 1 -- 172.21.15.83:0/948244945 <== mon.1 v2:172.21.15.83:3301/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7fc0540035f0 con 0x7fc058098d40 2024-09-14T03:40:44.989 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.987+0000 7fc04dffb640 1 -- 172.21.15.83:0/948244945 <== mon.1 v2:172.21.15.83:3301/0 5 ==== osd_map(33..33 src has 1..33) ==== 3027+0+0 (secure 0 0 0) 0x7fc054069960 con 0x7fc058098d40 2024-09-14T03:40:44.992 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:44.990+0000 7fc04dffb640 1 -- 172.21.15.83:0/948244945 <== mon.1 v2:172.21.15.83:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+157506 (secure 0 0 0) 0x7fc054068a10 con 0x7fc058098d40 2024-09-14T03:40:45.072 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.070+0000 7fbbcfe9a640 1 -- 172.21.15.83:0/2730178816 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 1} v 0) -- 0x7fbbc80698c0 con 0x7fbbc806edc0 2024-09-14T03:40:45.072 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.070+0000 7fbbbe7fc640 1 -- 172.21.15.83:0/2730178816 <== mon.1 v2:172.21.15.83:3301/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 1}]=0 v0) ==== 74+0+13 (secure 0 0 0) 0x7fbbb80687f0 con 0x7fbbc806edc0 2024-09-14T03:40:45.072 INFO:teuthology.orchestra.run.smithi083.stdout:115964116996 2024-09-14T03:40:45.074 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.072+0000 7fbbcfe9a640 1 -- 172.21.15.83:0/2730178816 >> v1:172.21.15.83:6800/1803897559 conn(0x7fbbac0514b0 legacy=0x7fbbac053970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:45.074 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.072+0000 7fbbcfe9a640 1 -- 172.21.15.83:0/2730178816 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fbbc806edc0 msgr2=0x7fbbc81c4fd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:45.074 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.072+0000 7fbbcfe9a640 1 --2- 172.21.15.83:0/2730178816 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fbbc806edc0 0x7fbbc81c4fd0 secure :-1 s=READY pgs=60 cs=0 l=1 rev1=1 crypto rx=0x7fbbb800e9f0 tx=0x7fbbb800eeb0 comp rx=0 tx=0).stop 2024-09-14T03:40:45.074 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.072+0000 7fbbcfe9a640 1 -- 172.21.15.83:0/2730178816 shutdown_connections 2024-09-14T03:40:45.074 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.072+0000 7fbbcfe9a640 1 --2- 172.21.15.83:0/2730178816 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fbbc81322e0 0x7fbbc81c5510 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:45.074 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.072+0000 7fbbcfe9a640 1 --2- 172.21.15.83:0/2730178816 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fbbc806edc0 0x7fbbc81c4fd0 unknown :-1 s=CLOSED pgs=60 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:45.074 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.072+0000 7fbbcfe9a640 1 --2- 172.21.15.83:0/2730178816 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fbbc806e2d0 0x7fbbc81c4a90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:45.074 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.072+0000 7fbbcfe9a640 1 -- 172.21.15.83:0/2730178816 >> 172.21.15.83:0/2730178816 conn(0x7fbbc809e490 msgr2=0x7fbbc8127380 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:45.075 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.072+0000 7fbbcfe9a640 1 -- 172.21.15.83:0/2730178816 shutdown_connections 2024-09-14T03:40:45.075 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.072+0000 7fbbcfe9a640 1 -- 172.21.15.83:0/2730178816 wait complete. 2024-09-14T03:40:45.076 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.074+0000 7f0419543640 1 -- 172.21.15.83:0/3988886264 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 0} v 0) -- 0x7f0414099a70 con 0x7f04140712e0 2024-09-14T03:40:45.076 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.075+0000 7f040b7fe640 1 -- 172.21.15.83:0/3988886264 <== mon.0 v2:172.21.15.83:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 0}]=0 v0) ==== 74+0+13 (secure 0 0 0) 0x7f0414099a70 con 0x7f04140712e0 2024-09-14T03:40:45.076 INFO:teuthology.orchestra.run.smithi083.stdout:115964116996 2024-09-14T03:40:45.078 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.077+0000 7f0419543640 1 -- 172.21.15.83:0/3988886264 >> v1:172.21.15.83:6800/1803897559 conn(0x7f03f40514b0 legacy=0x7f03f4053970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:45.078 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.077+0000 7f0419543640 1 -- 172.21.15.83:0/3988886264 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f04140712e0 msgr2=0x7f0414126690 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:45.079 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.077+0000 7f0419543640 1 --2- 172.21.15.83:0/3988886264 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f04140712e0 0x7f0414126690 secure :-1 s=READY pgs=104 cs=0 l=1 rev1=1 crypto rx=0x7f040c009bf0 tx=0x7f040c003ba0 comp rx=0 tx=0).stop 2024-09-14T03:40:45.079 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.077+0000 7f0419543640 1 -- 172.21.15.83:0/3988886264 shutdown_connections 2024-09-14T03:40:45.079 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.077+0000 7f0419543640 1 --2- 172.21.15.83:0/3988886264 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f04140712e0 0x7f0414126690 unknown :-1 s=CLOSED pgs=104 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:45.079 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.077+0000 7f0419543640 1 --2- 172.21.15.83:0/3988886264 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f0414070f00 0x7f0414126150 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:45.079 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.077+0000 7f0419543640 1 --2- 172.21.15.83:0/3988886264 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f04140705e0 0x7f0414125c10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:45.079 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.077+0000 7f0419543640 1 -- 172.21.15.83:0/3988886264 >> 172.21.15.83:0/3988886264 conn(0x7f041409dde0 msgr2=0x7f041412f390 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:45.079 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.077+0000 7f0419543640 1 -- 172.21.15.83:0/3988886264 shutdown_connections 2024-09-14T03:40:45.079 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.077+0000 7f0419543640 1 -- 172.21.15.83:0/3988886264 wait complete. 2024-09-14T03:40:45.086 INFO:tasks.ceph.ceph_manager.ceph:need seq 115964116996 got 115964116996 for osd.1 2024-09-14T03:40:45.087 DEBUG:teuthology.parallel:result is None 2024-09-14T03:40:45.089 INFO:tasks.ceph.ceph_manager.ceph:need seq 115964116996 got 115964116996 for osd.0 2024-09-14T03:40:45.090 DEBUG:teuthology.parallel:result is None 2024-09-14T03:40:45.112 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.110+0000 7f4aae41d640 1 -- 172.21.15.83:0/2361069685 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 2} v 0) -- 0x7f4aa80698c0 con 0x7f4aa8099660 2024-09-14T03:40:45.112 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.111+0000 7f4aa4ff9640 1 -- 172.21.15.83:0/2361069685 <== mon.2 v2:172.21.15.83:3302/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 2}]=0 v0) ==== 74+0+13 (secure 0 0 0) 0x7f4a9c089300 con 0x7f4aa8099660 2024-09-14T03:40:45.112 INFO:teuthology.orchestra.run.smithi083.stdout:128849018883 2024-09-14T03:40:45.114 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.112+0000 7f4aae41d640 1 -- 172.21.15.83:0/2361069685 >> v1:172.21.15.83:6800/1803897559 conn(0x7f4a940514b0 legacy=0x7f4a94053970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:45.114 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.112+0000 7f4aae41d640 1 -- 172.21.15.83:0/2361069685 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f4aa8099660 msgr2=0x7f4aa8071d40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:45.114 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.112+0000 7f4aae41d640 1 --2- 172.21.15.83:0/2361069685 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f4aa8099660 0x7f4aa8071d40 secure :-1 s=READY pgs=61 cs=0 l=1 rev1=1 crypto rx=0x7f4a9c009950 tx=0x7f4a9c032cd0 comp rx=0 tx=0).stop 2024-09-14T03:40:45.114 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.112+0000 7f4aae41d640 1 -- 172.21.15.83:0/2361069685 shutdown_connections 2024-09-14T03:40:45.114 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.112+0000 7f4aae41d640 1 --2- 172.21.15.83:0/2361069685 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f4aa812eff0 0x7f4aa8072280 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:45.114 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.112+0000 7f4aae41d640 1 --2- 172.21.15.83:0/2361069685 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f4aa8099660 0x7f4aa8071d40 unknown :-1 s=CLOSED pgs=61 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:45.114 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.112+0000 7f4aae41d640 1 --2- 172.21.15.83:0/2361069685 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f4aa8098d40 0x7f4aa8071800 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:45.114 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.112+0000 7f4aae41d640 1 -- 172.21.15.83:0/2361069685 >> 172.21.15.83:0/2361069685 conn(0x7f4aa80a1790 msgr2=0x7f4aa8128440 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:45.115 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.112+0000 7f4aae41d640 1 -- 172.21.15.83:0/2361069685 shutdown_connections 2024-09-14T03:40:45.115 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.112+0000 7f4aae41d640 1 -- 172.21.15.83:0/2361069685 wait complete. 2024-09-14T03:40:45.125 INFO:tasks.ceph.ceph_manager.ceph:need seq 128849018883 got 128849018883 for osd.2 2024-09-14T03:40:45.126 DEBUG:teuthology.parallel:result is None 2024-09-14T03:40:45.152 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.150+0000 7fc05f83e640 1 -- 172.21.15.83:0/948244945 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 3} v 0) -- 0x7fc058073eb0 con 0x7fc058098d40 2024-09-14T03:40:45.152 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.150+0000 7fc04dffb640 1 -- 172.21.15.83:0/948244945 <== mon.1 v2:172.21.15.83:3301/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 3}]=0 v0) ==== 74+0+13 (secure 0 0 0) 0x7fc054004400 con 0x7fc058098d40 2024-09-14T03:40:45.152 INFO:teuthology.orchestra.run.smithi083.stdout:124554051588 2024-09-14T03:40:45.154 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.152+0000 7fc05f83e640 1 -- 172.21.15.83:0/948244945 >> v1:172.21.15.83:6800/1803897559 conn(0x7fc03c0514b0 legacy=0x7fc03c053970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:45.154 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.153+0000 7fc05f83e640 1 -- 172.21.15.83:0/948244945 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fc058098d40 msgr2=0x7fc0581c4d50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:45.154 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.153+0000 7fc05f83e640 1 --2- 172.21.15.83:0/948244945 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fc058098d40 0x7fc0581c4d50 secure :-1 s=READY pgs=61 cs=0 l=1 rev1=1 crypto rx=0x7fc054005fd0 tx=0x7fc05400ee50 comp rx=0 tx=0).stop 2024-09-14T03:40:45.154 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.153+0000 7fc05f83e640 1 -- 172.21.15.83:0/948244945 shutdown_connections 2024-09-14T03:40:45.154 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.153+0000 7fc05f83e640 1 --2- 172.21.15.83:0/948244945 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fc05812eff0 0x7fc058071570 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:45.155 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.153+0000 7fc05f83e640 1 --2- 172.21.15.83:0/948244945 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fc058099660 0x7fc0581c5290 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:45.155 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.153+0000 7fc05f83e640 1 --2- 172.21.15.83:0/948244945 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fc058098d40 0x7fc0581c4d50 unknown :-1 s=CLOSED pgs=61 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:45.155 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.153+0000 7fc05f83e640 1 -- 172.21.15.83:0/948244945 >> 172.21.15.83:0/948244945 conn(0x7fc0580a1790 msgr2=0x7fc058128440 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:45.155 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.153+0000 7fc05f83e640 1 -- 172.21.15.83:0/948244945 shutdown_connections 2024-09-14T03:40:45.155 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.153+0000 7fc05f83e640 1 -- 172.21.15.83:0/948244945 wait complete. 2024-09-14T03:40:45.165 INFO:tasks.ceph.ceph_manager.ceph:need seq 124554051587 got 124554051588 for osd.3 2024-09-14T03:40:45.165 DEBUG:teuthology.parallel:result is None 2024-09-14T03:40:45.165 INFO:tasks.ceph.ceph_manager.ceph:waiting for clean 2024-09-14T03:40:45.166 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-09-14T03:40:45.305 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.303+0000 7f4832407640 1 -- 172.21.15.83:0/3149946294 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f482c0991f0 msgr2=0x7f482c12bc20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:45.305 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.303+0000 7f4832407640 1 --2- 172.21.15.83:0/3149946294 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f482c0991f0 0x7f482c12bc20 secure :-1 s=READY pgs=62 cs=0 l=1 rev1=1 crypto rx=0x7f481c009860 tx=0x7f481c02a420 comp rx=0 tx=0).stop 2024-09-14T03:40:45.305 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.303+0000 7f4832407640 1 -- 172.21.15.83:0/3149946294 shutdown_connections 2024-09-14T03:40:45.305 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.303+0000 7f4832407640 1 --2- 172.21.15.83:0/3149946294 >> v2:172.21.15.83:3300/0 conn(0x7f482c095df0 0x7f482c133b10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:45.305 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.303+0000 7f4832407640 1 --2- 172.21.15.83:0/3149946294 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f482c0991f0 0x7f482c12bc20 unknown :-1 s=CLOSED pgs=62 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:45.305 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.303+0000 7f4832407640 1 -- 172.21.15.83:0/3149946294 >> 172.21.15.83:0/3149946294 conn(0x7f482c09a2e0 msgr2=0x7f482c097ad0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:45.305 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.303+0000 7f4832407640 1 -- 172.21.15.83:0/3149946294 shutdown_connections 2024-09-14T03:40:45.305 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.303+0000 7f4832407640 1 -- 172.21.15.83:0/3149946294 wait complete. 2024-09-14T03:40:45.305 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.304+0000 7f4832407640 1 Processor -- start 2024-09-14T03:40:45.307 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.304+0000 7f4832407640 1 -- start start 2024-09-14T03:40:45.308 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.304+0000 7f4832407640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f482c095df0 0x7f482c0729f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:45.308 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.304+0000 7f4832407640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f482c0991f0 0x7f482c072f30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:45.308 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.304+0000 7f4832407640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f482c12c160 0x7f482c073470 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:45.308 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.304+0000 7f4832407640 1 -- --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7f482c1346e0 con 0x7f482c0991f0 2024-09-14T03:40:45.308 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.304+0000 7f4832407640 1 -- --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7f482c134560 con 0x7f482c12c160 2024-09-14T03:40:45.308 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.304+0000 7f4832407640 1 -- --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7f482c134860 con 0x7f482c095df0 2024-09-14T03:40:45.308 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.304+0000 7f482b7fe640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f482c0991f0 0x7f482c072f30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:45.308 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.304+0000 7f482bfff640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f482c095df0 0x7f482c0729f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:45.308 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.304+0000 7f48309a4640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f482c12c160 0x7f482c073470 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:45.308 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.304+0000 7f482b7fe640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f482c0991f0 0x7f482c072f30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3300/0 says I am v2:172.21.15.83:40058/0 (socket says 172.21.15.83:40058) 2024-09-14T03:40:45.308 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.304+0000 7f482b7fe640 1 -- 172.21.15.83:0/2379637143 learned_addr learned my addr 172.21.15.83:0/2379637143 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:40:45.308 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.305+0000 7f482b7fe640 1 -- 172.21.15.83:0/2379637143 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f482c095df0 msgr2=0x7f482c0729f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:45.308 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.305+0000 7f482b7fe640 1 --2- 172.21.15.83:0/2379637143 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f482c095df0 0x7f482c0729f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:45.309 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.305+0000 7f482b7fe640 1 -- 172.21.15.83:0/2379637143 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f482c12c160 msgr2=0x7f482c073470 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:45.309 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.305+0000 7f482b7fe640 1 --2- 172.21.15.83:0/2379637143 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f482c12c160 0x7f482c073470 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:45.309 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.305+0000 7f482b7fe640 1 -- 172.21.15.83:0/2379637143 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f482c128ed0 con 0x7f482c0991f0 2024-09-14T03:40:45.309 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.305+0000 7f48309a4640 1 --2- 172.21.15.83:0/2379637143 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f482c12c160 0x7f482c073470 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:40:45.309 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.305+0000 7f482b7fe640 1 --2- 172.21.15.83:0/2379637143 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f482c0991f0 0x7f482c072f30 secure :-1 s=READY pgs=110 cs=0 l=1 rev1=1 crypto rx=0x7f481400c500 tx=0x7f481400c9c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:40:45.309 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.305+0000 7f4828ff9640 1 -- 172.21.15.83:0/2379637143 <== mon.0 v2:172.21.15.83:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f48140052d0 con 0x7f482c0991f0 2024-09-14T03:40:45.309 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.305+0000 7f4828ff9640 1 -- 172.21.15.83:0/2379637143 <== mon.0 v2:172.21.15.83:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f4814005470 con 0x7f482c0991f0 2024-09-14T03:40:45.309 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.305+0000 7f4832407640 1 -- 172.21.15.83:0/2379637143 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f482c0953a0 con 0x7f482c0991f0 2024-09-14T03:40:45.309 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.305+0000 7f4828ff9640 1 -- 172.21.15.83:0/2379637143 <== mon.0 v2:172.21.15.83:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4814012870 con 0x7f482c0991f0 2024-09-14T03:40:45.309 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.305+0000 7f4832407640 1 -- 172.21.15.83:0/2379637143 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f482c094f00 con 0x7f482c0991f0 2024-09-14T03:40:45.309 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.306+0000 7f4832407640 1 -- 172.21.15.83:0/2379637143 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f482c12bc20 con 0x7f482c0991f0 2024-09-14T03:40:45.309 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.306+0000 7f4828ff9640 1 -- 172.21.15.83:0/2379637143 <== mon.0 v2:172.21.15.83:3300/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7f48140035f0 con 0x7f482c0991f0 2024-09-14T03:40:45.309 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.306+0000 7f4828ff9640 1 -- 172.21.15.83:0/2379637143 <== mon.0 v2:172.21.15.83:3300/0 5 ==== osd_map(33..33 src has 1..33) ==== 3027+0+0 (secure 0 0 0) 0x7f4814068b70 con 0x7f482c0991f0 2024-09-14T03:40:45.312 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.310+0000 7f4828ff9640 1 -- 172.21.15.83:0/2379637143 <== mon.0 v2:172.21.15.83:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+157506 (secure 0 0 0) 0x7f4814067c20 con 0x7f482c0991f0 2024-09-14T03:40:45.454 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.453+0000 7f4832407640 1 -- 172.21.15.83:0/2379637143 --> v1:172.21.15.83:6800/1803897559 -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) -- 0x7f482c0699c0 con 0x7f480c0514b0 2024-09-14T03:40:45.455 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.454+0000 7f4828ff9640 1 -- 172.21.15.83:0/2379637143 <== mgr.14106 v1:172.21.15.83:6800/1803897559 1 ==== mgr_command_reply(tid 0: 0 dumped all) ==== 18+0+32027 (unknown 2965378022 0 3614388528) 0x7f482c0699c0 con 0x7f480c0514b0 2024-09-14T03:40:45.456 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:40:45.456 INFO:teuthology.orchestra.run.smithi083.stderr:dumped all 2024-09-14T03:40:45.457 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.455+0000 7f4832407640 1 -- 172.21.15.83:0/2379637143 >> v1:172.21.15.83:6800/1803897559 conn(0x7f480c0514b0 legacy=0x7f480c053970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:45.458 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.455+0000 7f4832407640 1 -- 172.21.15.83:0/2379637143 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f482c0991f0 msgr2=0x7f482c072f30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:45.458 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.455+0000 7f4832407640 1 --2- 172.21.15.83:0/2379637143 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f482c0991f0 0x7f482c072f30 secure :-1 s=READY pgs=110 cs=0 l=1 rev1=1 crypto rx=0x7f481400c500 tx=0x7f481400c9c0 comp rx=0 tx=0).stop 2024-09-14T03:40:45.458 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.456+0000 7f4832407640 1 -- 172.21.15.83:0/2379637143 shutdown_connections 2024-09-14T03:40:45.458 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.456+0000 7f4832407640 1 --2- 172.21.15.83:0/2379637143 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f482c12c160 0x7f482c073470 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:45.458 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.456+0000 7f4832407640 1 --2- 172.21.15.83:0/2379637143 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f482c0991f0 0x7f482c072f30 unknown :-1 s=CLOSED pgs=110 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:45.458 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.456+0000 7f4832407640 1 --2- 172.21.15.83:0/2379637143 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f482c095df0 0x7f482c0729f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:45.458 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.456+0000 7f4832407640 1 -- 172.21.15.83:0/2379637143 >> 172.21.15.83:0/2379637143 conn(0x7f482c09a2e0 msgr2=0x7f482c08bf80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:45.458 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.456+0000 7f4832407640 1 -- 172.21.15.83:0/2379637143 shutdown_connections 2024-09-14T03:40:45.458 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.456+0000 7f4832407640 1 -- 172.21.15.83:0/2379637143 wait complete. 2024-09-14T03:40:45.468 INFO:teuthology.orchestra.run.smithi083.stdout:{"pg_ready":true,"pg_map":{"version":27,"stamp":"2024-09-14T03:40:44.043302+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":166,"num_read_kb":267,"num_write":151,"num_write_kb":1896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":107,"ondisk_log_size":107,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":4,"num_per_pool_osds":4,"num_per_pool_omap_osds":4,"kb":377487360,"kb_used":112356,"kb_used_data":2240,"kb_used_omap":17,"kb_used_meta":109998,"kb_avail":377375004,"statfs":{"total":386547056640,"available":386432004096,"internally_reserved":0,"allocated":2293760,"data_stored":1557188,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":17731,"internal_metadata":112638653},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"7.940031"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":54,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-14T03:40:41.119404+0000","last_change":"2024-09-14T03:40:35.310297+0000","last_active":"2024-09-14T03:40:41.119404+0000","last_peered":"2024-09-14T03:40:41.119404+0000","last_clean":"2024-09-14T03:40:41.119404+0000","last_became_active":"2024-09-14T03:40:35.309980+0000","last_became_peered":"2024-09-14T03:40:35.309980+0000","last_unstale":"2024-09-14T03:40:41.119404+0000","last_undegraded":"2024-09-14T03:40:41.119404+0000","last_fullsized":"2024-09-14T03:40:41.119404+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_clean_scrub_stamp":"2024-09-14T03:39:00.791161+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:33:56.556777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00098140899999999993,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":44,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-14T03:40:41.120097+0000","last_change":"2024-09-14T03:40:33.102548+0000","last_active":"2024-09-14T03:40:41.120097+0000","last_peered":"2024-09-14T03:40:41.120097+0000","last_clean":"2024-09-14T03:40:41.120097+0000","last_became_active":"2024-09-14T03:40:33.102279+0000","last_became_peered":"2024-09-14T03:40:33.102279+0000","last_unstale":"2024-09-14T03:40:41.120097+0000","last_undegraded":"2024-09-14T03:40:41.120097+0000","last_fullsized":"2024-09-14T03:40:41.120097+0000","mapping_epoch":27,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":28,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_clean_scrub_stamp":"2024-09-14T03:39:00.791161+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:26:35.820454+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00045411099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":54,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-14T03:40:41.119610+0000","last_change":"2024-09-14T03:40:35.309898+0000","last_active":"2024-09-14T03:40:41.119610+0000","last_peered":"2024-09-14T03:40:41.119610+0000","last_clean":"2024-09-14T03:40:41.119610+0000","last_became_active":"2024-09-14T03:40:35.309659+0000","last_became_peered":"2024-09-14T03:40:35.309659+0000","last_unstale":"2024-09-14T03:40:41.119610+0000","last_undegraded":"2024-09-14T03:40:41.119610+0000","last_fullsized":"2024-09-14T03:40:41.119610+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_clean_scrub_stamp":"2024-09-14T03:39:00.791161+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:46:05.387098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00087291699999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":44,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-14T03:40:41.120130+0000","last_change":"2024-09-14T03:40:33.102548+0000","last_active":"2024-09-14T03:40:41.120130+0000","last_peered":"2024-09-14T03:40:41.120130+0000","last_clean":"2024-09-14T03:40:41.120130+0000","last_became_active":"2024-09-14T03:40:33.102255+0000","last_became_peered":"2024-09-14T03:40:33.102255+0000","last_unstale":"2024-09-14T03:40:41.120130+0000","last_undegraded":"2024-09-14T03:40:41.120130+0000","last_fullsized":"2024-09-14T03:40:41.120130+0000","mapping_epoch":27,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":28,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_clean_scrub_stamp":"2024-09-14T03:39:00.791161+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:46:51.929694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00048052199999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"18'2","reported_seq":52,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-14T03:40:41.119384+0000","last_change":"2024-09-14T03:40:35.110955+0000","last_active":"2024-09-14T03:40:41.119384+0000","last_peered":"2024-09-14T03:40:41.119384+0000","last_clean":"2024-09-14T03:40:41.119384+0000","last_became_active":"2024-09-14T03:40:35.110637+0000","last_became_peered":"2024-09-14T03:40:35.110637+0000","last_unstale":"2024-09-14T03:40:41.119384+0000","last_undegraded":"2024-09-14T03:40:41.119384+0000","last_fullsized":"2024-09-14T03:40:41.119384+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_clean_scrub_stamp":"2024-09-14T03:39:00.791161+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:35:22.417821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00135844,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":37,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-14T03:40:41.177732+0000","last_change":"2024-09-14T03:40:36.107870+0000","last_active":"2024-09-14T03:40:41.177732+0000","last_peered":"2024-09-14T03:40:41.177732+0000","last_clean":"2024-09-14T03:40:41.177732+0000","last_became_active":"2024-09-14T03:40:36.107387+0000","last_became_peered":"2024-09-14T03:40:36.107387+0000","last_unstale":"2024-09-14T03:40:41.177732+0000","last_undegraded":"2024-09-14T03:40:41.177732+0000","last_fullsized":"2024-09-14T03:40:41.177732+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_clean_scrub_stamp":"2024-09-14T03:39:00.791161+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:24:43.135910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046880499999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":50,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-14T03:40:41.119371+0000","last_change":"2024-09-14T03:40:35.105432+0000","last_active":"2024-09-14T03:40:41.119371+0000","last_peered":"2024-09-14T03:40:41.119371+0000","last_clean":"2024-09-14T03:40:41.119371+0000","last_became_active":"2024-09-14T03:40:35.105125+0000","last_became_peered":"2024-09-14T03:40:35.105125+0000","last_unstale":"2024-09-14T03:40:41.119371+0000","last_undegraded":"2024-09-14T03:40:41.119371+0000","last_fullsized":"2024-09-14T03:40:41.119371+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_clean_scrub_stamp":"2024-09-14T03:39:00.791161+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:06:08.387860+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000765802,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"16'1","reported_seq":48,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-14T03:40:41.119494+0000","last_change":"2024-09-14T03:40:36.107793+0000","last_active":"2024-09-14T03:40:41.119494+0000","last_peered":"2024-09-14T03:40:41.119494+0000","last_clean":"2024-09-14T03:40:41.119494+0000","last_became_active":"2024-09-14T03:40:36.107394+0000","last_became_peered":"2024-09-14T03:40:36.107394+0000","last_unstale":"2024-09-14T03:40:41.119494+0000","last_undegraded":"2024-09-14T03:40:41.119494+0000","last_fullsized":"2024-09-14T03:40:41.119494+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_clean_scrub_stamp":"2024-09-14T03:39:00.791161+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:34:57.278411+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00075472199999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"21'104","reported_seq":199,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-14T03:40:41.119989+0000","last_change":"2024-09-14T03:40:33.097410+0000","last_active":"2024-09-14T03:40:41.119989+0000","last_peered":"2024-09-14T03:40:41.119989+0000","last_clean":"2024-09-14T03:40:41.119989+0000","last_became_active":"2024-09-14T03:40:33.097097+0000","last_became_peered":"2024-09-14T03:40:33.097097+0000","last_unstale":"2024-09-14T03:40:41.119989+0000","last_undegraded":"2024-09-14T03:40:41.119989+0000","last_fullsized":"2024-09-14T03:40:41.119989+0000","mapping_epoch":27,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":28,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:38:57.773685+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:57.773685+0000","last_clean_scrub_stamp":"2024-09-14T03:38:57.773685+0000","objects_scrubbed":0,"log_size":104,"log_dups_size":0,"ondisk_log_size":104,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:08:55.427294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":166,"num_read_kb":267,"num_write":149,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":4},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":166,"num_read_kb":267,"num_write":149,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":104,"ondisk_log_size":104,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":3,"up_from":29,"seq":124554051588,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27820,"kb_used_data":276,"kb_used_omap":4,"kb_used_meta":27515,"kb_avail":94344020,"statfs":{"total":96636764160,"available":96608276480,"internally_reserved":0,"allocated":282624,"data_stored":95913,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4433,"internal_metadata":28176047},"hb_peers":[0,1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":30,"seq":128849018883,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27804,"kb_used_data":256,"kb_used_omap":4,"kb_used_meta":27515,"kb_avail":94344036,"statfs":{"total":96636764160,"available":96608292864,"internally_reserved":0,"allocated":262144,"data_stored":88732,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4432,"internal_metadata":28176048},"hb_peers":[0,1,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":27,"seq":115964116996,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":28336,"kb_used_data":856,"kb_used_omap":4,"kb_used_meta":27451,"kb_avail":94343504,"statfs":{"total":96636764160,"available":96607748096,"internally_reserved":0,"allocated":876544,"data_stored":686281,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4431,"internal_metadata":28110513},"hb_peers":[0,2,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":27,"seq":115964116996,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":28396,"kb_used_data":852,"kb_used_omap":4,"kb_used_meta":27515,"kb_avail":94343444,"statfs":{"total":96636764160,"available":96607686656,"internally_reserved":0,"allocated":872448,"data_stored":686262,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4435,"internal_metadata":28176045},"hb_peers":[1,2,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T03:40:45.469 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-09-14T03:40:45.567 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.565+0000 7f521a06d640 1 -- 172.21.15.83:0/2448542564 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f5214095c20 msgr2=0x7f5214096060 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:45.567 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.565+0000 7f521a06d640 1 --2- 172.21.15.83:0/2448542564 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f5214095c20 0x7f5214096060 secure :-1 s=READY pgs=111 cs=0 l=1 rev1=1 crypto rx=0x7f520c009900 tx=0x7f520c02a440 comp rx=0 tx=0).stop 2024-09-14T03:40:45.567 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.565+0000 7f521a06d640 1 -- 172.21.15.83:0/2448542564 shutdown_connections 2024-09-14T03:40:45.568 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.565+0000 7f521a06d640 1 --2- 172.21.15.83:0/2448542564 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f5214095c20 0x7f5214096060 unknown :-1 s=CLOSED pgs=111 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:45.568 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.565+0000 7f521a06d640 1 --2- 172.21.15.83:0/2448542564 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f521412c160 0x7f52140956e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:45.568 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.565+0000 7f521a06d640 1 --2- 172.21.15.83:0/2448542564 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f52140991f0 0x7f521412bc20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:45.568 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.565+0000 7f521a06d640 1 -- 172.21.15.83:0/2448542564 >> 172.21.15.83:0/2448542564 conn(0x7f521409a2e0 msgr2=0x7f5214097ad0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:45.568 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.565+0000 7f521a06d640 1 -- 172.21.15.83:0/2448542564 shutdown_connections 2024-09-14T03:40:45.568 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.565+0000 7f521a06d640 1 -- 172.21.15.83:0/2448542564 wait complete. 2024-09-14T03:40:45.568 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.566+0000 7f521a06d640 1 Processor -- start 2024-09-14T03:40:45.570 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.566+0000 7f521a06d640 1 -- start start 2024-09-14T03:40:45.571 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.567+0000 7f521a06d640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f5214095c20 0x7f52141c2b70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:45.571 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.567+0000 7f521a06d640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f52140991f0 0x7f52141c30b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:45.571 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.567+0000 7f521a06d640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f521412c160 0x7f521412f7e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:45.571 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.567+0000 7f521a06d640 1 -- --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7f5214133490 con 0x7f5214095c20 2024-09-14T03:40:45.571 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.567+0000 7f521a06d640 1 -- --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7f5214133310 con 0x7f52140991f0 2024-09-14T03:40:45.571 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.567+0000 7f521a06d640 1 -- --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7f5214133610 con 0x7f521412c160 2024-09-14T03:40:45.571 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.567+0000 7f52137fe640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f5214095c20 0x7f52141c2b70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:45.571 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.567+0000 7f5212ffd640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f52140991f0 0x7f52141c30b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:45.571 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.567+0000 7f52137fe640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f5214095c20 0x7f52141c2b70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3300/0 says I am v2:172.21.15.83:40070/0 (socket says 172.21.15.83:40070) 2024-09-14T03:40:45.572 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.567+0000 7f5213fff640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f521412c160 0x7f521412f7e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:45.572 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.567+0000 7f52137fe640 1 -- 172.21.15.83:0/3533454432 learned_addr learned my addr 172.21.15.83:0/3533454432 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:40:45.572 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.567+0000 7f52137fe640 1 -- 172.21.15.83:0/3533454432 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f521412c160 msgr2=0x7f521412f7e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:45.572 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.567+0000 7f52137fe640 1 --2- 172.21.15.83:0/3533454432 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f521412c160 0x7f521412f7e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:45.572 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.567+0000 7f52137fe640 1 -- 172.21.15.83:0/3533454432 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f52140991f0 msgr2=0x7f52141c30b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:45.572 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.567+0000 7f52137fe640 1 --2- 172.21.15.83:0/3533454432 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f52140991f0 0x7f52141c30b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:45.572 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.567+0000 7f52137fe640 1 -- 172.21.15.83:0/3533454432 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f5214094030 con 0x7f5214095c20 2024-09-14T03:40:45.572 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.567+0000 7f5212ffd640 1 --2- 172.21.15.83:0/3533454432 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f52140991f0 0x7f52141c30b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:40:45.572 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.567+0000 7f52137fe640 1 --2- 172.21.15.83:0/3533454432 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f5214095c20 0x7f52141c2b70 secure :-1 s=READY pgs=112 cs=0 l=1 rev1=1 crypto rx=0x7f5204009d50 tx=0x7f520400c6a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:40:45.572 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.567+0000 7f51fbfff640 1 -- 172.21.15.83:0/3533454432 <== mon.0 v2:172.21.15.83:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5204003b70 con 0x7f5214095c20 2024-09-14T03:40:45.572 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.567+0000 7f51fbfff640 1 -- 172.21.15.83:0/3533454432 <== mon.0 v2:172.21.15.83:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f5204003d10 con 0x7f5214095c20 2024-09-14T03:40:45.572 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.567+0000 7f51fbfff640 1 -- 172.21.15.83:0/3533454432 <== mon.0 v2:172.21.15.83:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5204004280 con 0x7f5214095c20 2024-09-14T03:40:45.572 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.567+0000 7f521a06d640 1 -- 172.21.15.83:0/3533454432 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f521406eb20 con 0x7f5214095c20 2024-09-14T03:40:45.573 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.567+0000 7f521a06d640 1 -- 172.21.15.83:0/3533454432 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f52141c35f0 con 0x7f5214095c20 2024-09-14T03:40:45.573 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.568+0000 7f51fbfff640 1 -- 172.21.15.83:0/3533454432 <== mon.0 v2:172.21.15.83:3300/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7f5204004420 con 0x7f5214095c20 2024-09-14T03:40:45.573 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.568+0000 7f521a06d640 1 -- 172.21.15.83:0/3533454432 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f521412bc20 con 0x7f5214095c20 2024-09-14T03:40:45.573 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.569+0000 7f51fbfff640 1 -- 172.21.15.83:0/3533454432 <== mon.0 v2:172.21.15.83:3300/0 5 ==== osd_map(33..33 src has 1..33) ==== 3027+0+0 (secure 0 0 0) 0x7f5204069ce0 con 0x7f5214095c20 2024-09-14T03:40:45.574 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.572+0000 7f51fbfff640 1 -- 172.21.15.83:0/3533454432 <== mon.0 v2:172.21.15.83:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+157506 (secure 0 0 0) 0x7f5204068d90 con 0x7f5214095c20 2024-09-14T03:40:45.718 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.716+0000 7f521a06d640 1 -- 172.21.15.83:0/3533454432 --> v1:172.21.15.83:6800/1803897559 -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) -- 0x7f52140699c0 con 0x7f52000514b0 2024-09-14T03:40:45.718 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.717+0000 7f51fbfff640 1 -- 172.21.15.83:0/3533454432 <== mgr.14106 v1:172.21.15.83:6800/1803897559 1 ==== mgr_command_reply(tid 0: 0 dumped all) ==== 18+0+32027 (unknown 2965378022 0 3614388528) 0x7f52140699c0 con 0x7f52000514b0 2024-09-14T03:40:45.719 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:40:45.719 INFO:teuthology.orchestra.run.smithi083.stderr:dumped all 2024-09-14T03:40:45.721 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.719+0000 7f521a06d640 1 -- 172.21.15.83:0/3533454432 >> v1:172.21.15.83:6800/1803897559 conn(0x7f52000514b0 legacy=0x7f5200053970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:45.721 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.719+0000 7f521a06d640 1 -- 172.21.15.83:0/3533454432 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f5214095c20 msgr2=0x7f52141c2b70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:45.721 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.719+0000 7f521a06d640 1 --2- 172.21.15.83:0/3533454432 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f5214095c20 0x7f52141c2b70 secure :-1 s=READY pgs=112 cs=0 l=1 rev1=1 crypto rx=0x7f5204009d50 tx=0x7f520400c6a0 comp rx=0 tx=0).stop 2024-09-14T03:40:45.721 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.719+0000 7f521a06d640 1 -- 172.21.15.83:0/3533454432 shutdown_connections 2024-09-14T03:40:45.721 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.719+0000 7f521a06d640 1 --2- 172.21.15.83:0/3533454432 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f521412c160 0x7f521412f7e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:45.721 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.719+0000 7f521a06d640 1 --2- 172.21.15.83:0/3533454432 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f52140991f0 0x7f52141c30b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:45.721 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.719+0000 7f521a06d640 1 --2- 172.21.15.83:0/3533454432 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f5214095c20 0x7f52141c2b70 unknown :-1 s=CLOSED pgs=112 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:45.721 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.719+0000 7f521a06d640 1 -- 172.21.15.83:0/3533454432 >> 172.21.15.83:0/3533454432 conn(0x7f521409a2e0 msgr2=0x7f5214092b40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:45.721 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.719+0000 7f521a06d640 1 -- 172.21.15.83:0/3533454432 shutdown_connections 2024-09-14T03:40:45.721 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.719+0000 7f521a06d640 1 -- 172.21.15.83:0/3533454432 wait complete. 2024-09-14T03:40:45.731 INFO:teuthology.orchestra.run.smithi083.stdout:{"pg_ready":true,"pg_map":{"version":27,"stamp":"2024-09-14T03:40:44.043302+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":166,"num_read_kb":267,"num_write":151,"num_write_kb":1896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":107,"ondisk_log_size":107,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":4,"num_per_pool_osds":4,"num_per_pool_omap_osds":4,"kb":377487360,"kb_used":112356,"kb_used_data":2240,"kb_used_omap":17,"kb_used_meta":109998,"kb_avail":377375004,"statfs":{"total":386547056640,"available":386432004096,"internally_reserved":0,"allocated":2293760,"data_stored":1557188,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":17731,"internal_metadata":112638653},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"7.940031"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":54,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-14T03:40:41.119404+0000","last_change":"2024-09-14T03:40:35.310297+0000","last_active":"2024-09-14T03:40:41.119404+0000","last_peered":"2024-09-14T03:40:41.119404+0000","last_clean":"2024-09-14T03:40:41.119404+0000","last_became_active":"2024-09-14T03:40:35.309980+0000","last_became_peered":"2024-09-14T03:40:35.309980+0000","last_unstale":"2024-09-14T03:40:41.119404+0000","last_undegraded":"2024-09-14T03:40:41.119404+0000","last_fullsized":"2024-09-14T03:40:41.119404+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_clean_scrub_stamp":"2024-09-14T03:39:00.791161+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:33:56.556777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00098140899999999993,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":44,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-14T03:40:41.120097+0000","last_change":"2024-09-14T03:40:33.102548+0000","last_active":"2024-09-14T03:40:41.120097+0000","last_peered":"2024-09-14T03:40:41.120097+0000","last_clean":"2024-09-14T03:40:41.120097+0000","last_became_active":"2024-09-14T03:40:33.102279+0000","last_became_peered":"2024-09-14T03:40:33.102279+0000","last_unstale":"2024-09-14T03:40:41.120097+0000","last_undegraded":"2024-09-14T03:40:41.120097+0000","last_fullsized":"2024-09-14T03:40:41.120097+0000","mapping_epoch":27,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":28,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_clean_scrub_stamp":"2024-09-14T03:39:00.791161+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:26:35.820454+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00045411099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":54,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-14T03:40:41.119610+0000","last_change":"2024-09-14T03:40:35.309898+0000","last_active":"2024-09-14T03:40:41.119610+0000","last_peered":"2024-09-14T03:40:41.119610+0000","last_clean":"2024-09-14T03:40:41.119610+0000","last_became_active":"2024-09-14T03:40:35.309659+0000","last_became_peered":"2024-09-14T03:40:35.309659+0000","last_unstale":"2024-09-14T03:40:41.119610+0000","last_undegraded":"2024-09-14T03:40:41.119610+0000","last_fullsized":"2024-09-14T03:40:41.119610+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_clean_scrub_stamp":"2024-09-14T03:39:00.791161+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:46:05.387098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00087291699999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":44,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-14T03:40:41.120130+0000","last_change":"2024-09-14T03:40:33.102548+0000","last_active":"2024-09-14T03:40:41.120130+0000","last_peered":"2024-09-14T03:40:41.120130+0000","last_clean":"2024-09-14T03:40:41.120130+0000","last_became_active":"2024-09-14T03:40:33.102255+0000","last_became_peered":"2024-09-14T03:40:33.102255+0000","last_unstale":"2024-09-14T03:40:41.120130+0000","last_undegraded":"2024-09-14T03:40:41.120130+0000","last_fullsized":"2024-09-14T03:40:41.120130+0000","mapping_epoch":27,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":28,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_clean_scrub_stamp":"2024-09-14T03:39:00.791161+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:46:51.929694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00048052199999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"18'2","reported_seq":52,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-14T03:40:41.119384+0000","last_change":"2024-09-14T03:40:35.110955+0000","last_active":"2024-09-14T03:40:41.119384+0000","last_peered":"2024-09-14T03:40:41.119384+0000","last_clean":"2024-09-14T03:40:41.119384+0000","last_became_active":"2024-09-14T03:40:35.110637+0000","last_became_peered":"2024-09-14T03:40:35.110637+0000","last_unstale":"2024-09-14T03:40:41.119384+0000","last_undegraded":"2024-09-14T03:40:41.119384+0000","last_fullsized":"2024-09-14T03:40:41.119384+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_clean_scrub_stamp":"2024-09-14T03:39:00.791161+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:35:22.417821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00135844,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":37,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-14T03:40:41.177732+0000","last_change":"2024-09-14T03:40:36.107870+0000","last_active":"2024-09-14T03:40:41.177732+0000","last_peered":"2024-09-14T03:40:41.177732+0000","last_clean":"2024-09-14T03:40:41.177732+0000","last_became_active":"2024-09-14T03:40:36.107387+0000","last_became_peered":"2024-09-14T03:40:36.107387+0000","last_unstale":"2024-09-14T03:40:41.177732+0000","last_undegraded":"2024-09-14T03:40:41.177732+0000","last_fullsized":"2024-09-14T03:40:41.177732+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_clean_scrub_stamp":"2024-09-14T03:39:00.791161+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:24:43.135910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046880499999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":50,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-14T03:40:41.119371+0000","last_change":"2024-09-14T03:40:35.105432+0000","last_active":"2024-09-14T03:40:41.119371+0000","last_peered":"2024-09-14T03:40:41.119371+0000","last_clean":"2024-09-14T03:40:41.119371+0000","last_became_active":"2024-09-14T03:40:35.105125+0000","last_became_peered":"2024-09-14T03:40:35.105125+0000","last_unstale":"2024-09-14T03:40:41.119371+0000","last_undegraded":"2024-09-14T03:40:41.119371+0000","last_fullsized":"2024-09-14T03:40:41.119371+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_clean_scrub_stamp":"2024-09-14T03:39:00.791161+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:06:08.387860+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000765802,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"16'1","reported_seq":48,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-14T03:40:41.119494+0000","last_change":"2024-09-14T03:40:36.107793+0000","last_active":"2024-09-14T03:40:41.119494+0000","last_peered":"2024-09-14T03:40:41.119494+0000","last_clean":"2024-09-14T03:40:41.119494+0000","last_became_active":"2024-09-14T03:40:36.107394+0000","last_became_peered":"2024-09-14T03:40:36.107394+0000","last_unstale":"2024-09-14T03:40:41.119494+0000","last_undegraded":"2024-09-14T03:40:41.119494+0000","last_fullsized":"2024-09-14T03:40:41.119494+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_clean_scrub_stamp":"2024-09-14T03:39:00.791161+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:34:57.278411+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00075472199999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"21'104","reported_seq":199,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-14T03:40:41.119989+0000","last_change":"2024-09-14T03:40:33.097410+0000","last_active":"2024-09-14T03:40:41.119989+0000","last_peered":"2024-09-14T03:40:41.119989+0000","last_clean":"2024-09-14T03:40:41.119989+0000","last_became_active":"2024-09-14T03:40:33.097097+0000","last_became_peered":"2024-09-14T03:40:33.097097+0000","last_unstale":"2024-09-14T03:40:41.119989+0000","last_undegraded":"2024-09-14T03:40:41.119989+0000","last_fullsized":"2024-09-14T03:40:41.119989+0000","mapping_epoch":27,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":28,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:38:57.773685+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:57.773685+0000","last_clean_scrub_stamp":"2024-09-14T03:38:57.773685+0000","objects_scrubbed":0,"log_size":104,"log_dups_size":0,"ondisk_log_size":104,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:08:55.427294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":166,"num_read_kb":267,"num_write":149,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":4},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":166,"num_read_kb":267,"num_write":149,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":104,"ondisk_log_size":104,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":3,"up_from":29,"seq":124554051588,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27820,"kb_used_data":276,"kb_used_omap":4,"kb_used_meta":27515,"kb_avail":94344020,"statfs":{"total":96636764160,"available":96608276480,"internally_reserved":0,"allocated":282624,"data_stored":95913,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4433,"internal_metadata":28176047},"hb_peers":[0,1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":30,"seq":128849018883,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27804,"kb_used_data":256,"kb_used_omap":4,"kb_used_meta":27515,"kb_avail":94344036,"statfs":{"total":96636764160,"available":96608292864,"internally_reserved":0,"allocated":262144,"data_stored":88732,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4432,"internal_metadata":28176048},"hb_peers":[0,1,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":27,"seq":115964116996,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":28336,"kb_used_data":856,"kb_used_omap":4,"kb_used_meta":27451,"kb_avail":94343504,"statfs":{"total":96636764160,"available":96607748096,"internally_reserved":0,"allocated":876544,"data_stored":686281,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4431,"internal_metadata":28110513},"hb_peers":[0,2,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":27,"seq":115964116996,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":28396,"kb_used_data":852,"kb_used_omap":4,"kb_used_meta":27515,"kb_avail":94343444,"statfs":{"total":96636764160,"available":96607686656,"internally_reserved":0,"allocated":872448,"data_stored":686262,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4435,"internal_metadata":28176045},"hb_peers":[1,2,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T03:40:45.733 INFO:tasks.ceph.ceph_manager.ceph:clean! 2024-09-14T03:40:45.733 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-09-14T03:40:45.733 INFO:tasks.ceph.ceph_manager.ceph:wait_until_healthy 2024-09-14T03:40:45.733 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph health --format=json 2024-09-14T03:40:45.832 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.830+0000 7fe393171640 1 -- 172.21.15.83:0/2964272283 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fe38c12f1c0 msgr2=0x7fe38c12f610 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:45.832 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.830+0000 7fe393171640 1 --2- 172.21.15.83:0/2964272283 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fe38c12f1c0 0x7fe38c12f610 secure :-1 s=READY pgs=62 cs=0 l=1 rev1=1 crypto rx=0x7fe384009830 tx=0x7fe38402a480 comp rx=0 tx=0).stop 2024-09-14T03:40:45.832 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.830+0000 7fe393171640 1 -- 172.21.15.83:0/2964272283 shutdown_connections 2024-09-14T03:40:45.832 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.830+0000 7fe393171640 1 --2- 172.21.15.83:0/2964272283 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fe38c12f1c0 0x7fe38c12f610 unknown :-1 s=CLOSED pgs=62 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:45.832 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.830+0000 7fe393171640 1 --2- 172.21.15.83:0/2964272283 >> v2:172.21.15.83:3300/0 conn(0x7fe38c099080 0x7fe38c099460 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:45.832 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.830+0000 7fe393171640 1 -- 172.21.15.83:0/2964272283 >> 172.21.15.83:0/2964272283 conn(0x7fe38c0a1800 msgr2=0x7fe38c0a1c10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:45.833 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.830+0000 7fe393171640 1 -- 172.21.15.83:0/2964272283 shutdown_connections 2024-09-14T03:40:45.833 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.830+0000 7fe393171640 1 -- 172.21.15.83:0/2964272283 wait complete. 2024-09-14T03:40:45.833 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.830+0000 7fe393171640 1 Processor -- start 2024-09-14T03:40:45.833 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.830+0000 7fe393171640 1 -- start start 2024-09-14T03:40:45.835 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.831+0000 7fe393171640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fe38c099080 0x7fe38c0701e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:45.835 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.831+0000 7fe393171640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fe38c0999a0 0x7fe38c070720 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:45.835 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.831+0000 7fe393171640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fe38c12f1c0 0x7fe38c070c60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:45.835 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.831+0000 7fe393171640 1 -- --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7fe38c137660 con 0x7fe38c12f1c0 2024-09-14T03:40:45.835 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.831+0000 7fe393171640 1 -- --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7fe38c1374e0 con 0x7fe38c0999a0 2024-09-14T03:40:45.835 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.831+0000 7fe393171640 1 -- --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7fe38c1377e0 con 0x7fe38c099080 2024-09-14T03:40:45.836 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.831+0000 7fe390f0d640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fe38c099080 0x7fe38c0701e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:45.836 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.831+0000 7fe39170e640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fe38c12f1c0 0x7fe38c070c60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:45.836 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.831+0000 7fe38bfff640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fe38c0999a0 0x7fe38c070720 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:45.836 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.831+0000 7fe390f0d640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fe38c099080 0x7fe38c0701e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3302/0 says I am v2:172.21.15.83:60622/0 (socket says 172.21.15.83:60622) 2024-09-14T03:40:45.836 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.831+0000 7fe390f0d640 1 -- 172.21.15.83:0/3060548896 learned_addr learned my addr 172.21.15.83:0/3060548896 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:40:45.836 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.831+0000 7fe39170e640 1 -- 172.21.15.83:0/3060548896 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fe38c099080 msgr2=0x7fe38c0701e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:45.836 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.831+0000 7fe39170e640 1 --2- 172.21.15.83:0/3060548896 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fe38c099080 0x7fe38c0701e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:45.836 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.831+0000 7fe39170e640 1 -- 172.21.15.83:0/3060548896 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fe38c0999a0 msgr2=0x7fe38c070720 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:45.836 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.831+0000 7fe39170e640 1 --2- 172.21.15.83:0/3060548896 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fe38c0999a0 0x7fe38c070720 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:45.836 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.831+0000 7fe39170e640 1 -- 172.21.15.83:0/3060548896 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fe38c071ab0 con 0x7fe38c12f1c0 2024-09-14T03:40:45.836 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.831+0000 7fe390f0d640 1 --2- 172.21.15.83:0/3060548896 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fe38c099080 0x7fe38c0701e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:40:45.836 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.832+0000 7fe39170e640 1 --2- 172.21.15.83:0/3060548896 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fe38c12f1c0 0x7fe38c070c60 secure :-1 s=READY pgs=114 cs=0 l=1 rev1=1 crypto rx=0x7fe384031000 tx=0x7fe384032b90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:40:45.837 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.832+0000 7fe3897fa640 1 -- 172.21.15.83:0/3060548896 <== mon.0 v2:172.21.15.83:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe384040070 con 0x7fe38c12f1c0 2024-09-14T03:40:45.837 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.832+0000 7fe3897fa640 1 -- 172.21.15.83:0/3060548896 <== mon.0 v2:172.21.15.83:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fe384005310 con 0x7fe38c12f1c0 2024-09-14T03:40:45.837 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.832+0000 7fe393171640 1 -- 172.21.15.83:0/3060548896 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fe38c074280 con 0x7fe38c12f1c0 2024-09-14T03:40:45.837 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.832+0000 7fe3897fa640 1 -- 172.21.15.83:0/3060548896 <== mon.0 v2:172.21.15.83:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe3840047a0 con 0x7fe38c12f1c0 2024-09-14T03:40:45.837 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.832+0000 7fe393171640 1 -- 172.21.15.83:0/3060548896 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fe38c072e50 con 0x7fe38c12f1c0 2024-09-14T03:40:45.837 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.833+0000 7fe3897fa640 1 -- 172.21.15.83:0/3060548896 <== mon.0 v2:172.21.15.83:3300/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7fe384039040 con 0x7fe38c12f1c0 2024-09-14T03:40:45.837 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.833+0000 7fe393171640 1 -- 172.21.15.83:0/3060548896 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fe38c099460 con 0x7fe38c12f1c0 2024-09-14T03:40:45.837 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.833+0000 7fe3897fa640 1 -- 172.21.15.83:0/3060548896 <== mon.0 v2:172.21.15.83:3300/0 5 ==== osd_map(33..33 src has 1..33) ==== 3027+0+0 (secure 0 0 0) 0x7fe384089f30 con 0x7fe38c12f1c0 2024-09-14T03:40:45.838 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:45.837+0000 7fe3897fa640 1 -- 172.21.15.83:0/3060548896 <== mon.0 v2:172.21.15.83:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+157506 (secure 0 0 0) 0x7fe384088fe0 con 0x7fe38c12f1c0 2024-09-14T03:40:46.006 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.004+0000 7fe393171640 1 -- 172.21.15.83:0/3060548896 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_command({"prefix": "health", "format": "json"} v 0) -- 0x7fe38c0698c0 con 0x7fe38c12f1c0 2024-09-14T03:40:46.006 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.005+0000 7fe3897fa640 1 -- 172.21.15.83:0/3060548896 <== mon.0 v2:172.21.15.83:3300/0 7 ==== mon_command_ack([{"prefix": "health", "format": "json"}]=0 v0) ==== 72+0+46 (secure 0 0 0) 0x7fe38403fd50 con 0x7fe38c12f1c0 2024-09-14T03:40:46.006 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:40:46.006 INFO:teuthology.orchestra.run.smithi083.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-09-14T03:40:46.008 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.006+0000 7fe393171640 1 -- 172.21.15.83:0/3060548896 >> v1:172.21.15.83:6800/1803897559 conn(0x7fe36c0514b0 legacy=0x7fe36c053970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:46.008 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.006+0000 7fe393171640 1 -- 172.21.15.83:0/3060548896 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fe38c12f1c0 msgr2=0x7fe38c070c60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:46.008 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.006+0000 7fe393171640 1 --2- 172.21.15.83:0/3060548896 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fe38c12f1c0 0x7fe38c070c60 secure :-1 s=READY pgs=114 cs=0 l=1 rev1=1 crypto rx=0x7fe384031000 tx=0x7fe384032b90 comp rx=0 tx=0).stop 2024-09-14T03:40:46.008 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.006+0000 7fe393171640 1 -- 172.21.15.83:0/3060548896 shutdown_connections 2024-09-14T03:40:46.008 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.006+0000 7fe393171640 1 --2- 172.21.15.83:0/3060548896 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fe38c12f1c0 0x7fe38c070c60 unknown :-1 s=CLOSED pgs=114 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:46.008 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.006+0000 7fe393171640 1 --2- 172.21.15.83:0/3060548896 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fe38c0999a0 0x7fe38c070720 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:46.008 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.006+0000 7fe393171640 1 --2- 172.21.15.83:0/3060548896 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fe38c099080 0x7fe38c0701e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:46.009 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.006+0000 7fe393171640 1 -- 172.21.15.83:0/3060548896 >> 172.21.15.83:0/3060548896 conn(0x7fe38c0a1800 msgr2=0x7fe38c097940 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:46.009 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.006+0000 7fe393171640 1 -- 172.21.15.83:0/3060548896 shutdown_connections 2024-09-14T03:40:46.009 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.006+0000 7fe393171640 1 -- 172.21.15.83:0/3060548896 wait complete. 2024-09-14T03:40:46.019 INFO:tasks.ceph.ceph_manager.ceph:wait_until_healthy done 2024-09-14T03:40:46.019 INFO:teuthology.run_tasks:Running task print... 2024-09-14T03:40:46.029 INFO:teuthology.task.print:**** done ceph.restart 2024-09-14T03:40:46.030 DEBUG:teuthology.run_tasks:Unwinding manager ceph.restart 2024-09-14T03:40:46.069 DEBUG:teuthology.run_tasks:Unwinding manager ceph.restart 2024-09-14T03:40:46.119 DEBUG:teuthology.run_tasks:Unwinding manager install.upgrade 2024-09-14T03:40:46.169 DEBUG:teuthology.run_tasks:Unwinding manager ceph 2024-09-14T03:40:46.219 INFO:tasks.ceph.ceph_manager.ceph:waiting for clean 2024-09-14T03:40:46.220 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-09-14T03:40:46.320 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.318+0000 7faf5e9e7640 1 -- 172.21.15.83:0/2570171431 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7faf580945d0 msgr2=0x7faf580949d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:46.320 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.318+0000 7faf5e9e7640 1 --2- 172.21.15.83:0/2570171431 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7faf580945d0 0x7faf580949d0 secure :-1 s=READY pgs=63 cs=0 l=1 rev1=1 crypto rx=0x7faf48005fd0 tx=0x7faf4802a410 comp rx=0 tx=0).stop 2024-09-14T03:40:46.320 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.318+0000 7faf5e9e7640 1 -- 172.21.15.83:0/2570171431 shutdown_connections 2024-09-14T03:40:46.321 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.318+0000 7faf5e9e7640 1 --2- 172.21.15.83:0/2570171431 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7faf580945d0 0x7faf580949d0 unknown :-1 s=CLOSED pgs=63 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:46.321 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.318+0000 7faf5e9e7640 1 --2- 172.21.15.83:0/2570171431 >> v2:172.21.15.83:3300/0 conn(0x7faf58094f10 0x7faf58133c00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:46.321 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.318+0000 7faf5e9e7640 1 --2- 172.21.15.83:0/2570171431 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7faf5812cc30 0x7faf5812d010 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:46.321 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.318+0000 7faf5e9e7640 1 -- 172.21.15.83:0/2570171431 >> 172.21.15.83:0/2570171431 conn(0x7faf5809a2e0 msgr2=0x7faf58097ad0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:46.321 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.318+0000 7faf5e9e7640 1 -- 172.21.15.83:0/2570171431 shutdown_connections 2024-09-14T03:40:46.321 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.318+0000 7faf5e9e7640 1 -- 172.21.15.83:0/2570171431 wait complete. 2024-09-14T03:40:46.323 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.319+0000 7faf5e9e7640 1 Processor -- start 2024-09-14T03:40:46.323 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.319+0000 7faf5e9e7640 1 -- start start 2024-09-14T03:40:46.323 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.320+0000 7faf5e9e7640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7faf580945d0 0x7faf58130650 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:46.323 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.320+0000 7faf5e9e7640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7faf58094f10 0x7faf58130b90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:46.324 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.320+0000 7faf5e9e7640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7faf5812cc30 0x7faf581310d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:46.324 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.320+0000 7faf5e9e7640 1 -- --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7faf58134610 con 0x7faf580945d0 2024-09-14T03:40:46.324 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.320+0000 7faf5e9e7640 1 -- --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7faf58134490 con 0x7faf5812cc30 2024-09-14T03:40:46.324 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.320+0000 7faf5e9e7640 1 -- --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7faf58134790 con 0x7faf58094f10 2024-09-14T03:40:46.324 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.320+0000 7faf57fff640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7faf580945d0 0x7faf58130650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:46.324 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.320+0000 7faf577fe640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7faf58094f10 0x7faf58130b90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:46.324 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.320+0000 7faf5cf84640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7faf5812cc30 0x7faf581310d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:46.324 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.320+0000 7faf57fff640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7faf580945d0 0x7faf58130650 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3300/0 says I am v2:172.21.15.83:40116/0 (socket says 172.21.15.83:40116) 2024-09-14T03:40:46.324 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.320+0000 7faf57fff640 1 -- 172.21.15.83:0/1992837672 learned_addr learned my addr 172.21.15.83:0/1992837672 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:40:46.324 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.320+0000 7faf577fe640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7faf58094f10 0x7faf58130b90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3302/0 says I am v2:172.21.15.83:60646/0 (socket says 172.21.15.83:60646) 2024-09-14T03:40:46.325 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.320+0000 7faf577fe640 1 -- 172.21.15.83:0/1992837672 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7faf5812cc30 msgr2=0x7faf581310d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:46.325 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.320+0000 7faf577fe640 1 --2- 172.21.15.83:0/1992837672 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7faf5812cc30 0x7faf581310d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:46.325 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.320+0000 7faf577fe640 1 -- 172.21.15.83:0/1992837672 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7faf580945d0 msgr2=0x7faf58130650 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:46.325 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.320+0000 7faf577fe640 1 --2- 172.21.15.83:0/1992837672 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7faf580945d0 0x7faf58130650 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:46.325 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.320+0000 7faf577fe640 1 -- 172.21.15.83:0/1992837672 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7faf581225b0 con 0x7faf58094f10 2024-09-14T03:40:46.325 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.320+0000 7faf57fff640 1 --2- 172.21.15.83:0/1992837672 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7faf580945d0 0x7faf58130650 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:40:46.325 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.320+0000 7faf5cf84640 1 --2- 172.21.15.83:0/1992837672 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7faf5812cc30 0x7faf581310d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:40:46.325 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.320+0000 7faf577fe640 1 --2- 172.21.15.83:0/1992837672 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7faf58094f10 0x7faf58130b90 secure :-1 s=READY pgs=63 cs=0 l=1 rev1=1 crypto rx=0x7faf48031000 tx=0x7faf480038a0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:40:46.325 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.321+0000 7faf54ff9640 1 -- 172.21.15.83:0/1992837672 <== mon.2 v2:172.21.15.83:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7faf4803f070 con 0x7faf58094f10 2024-09-14T03:40:46.325 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.321+0000 7faf54ff9640 1 -- 172.21.15.83:0/1992837672 <== mon.2 v2:172.21.15.83:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7faf480047a0 con 0x7faf58094f10 2024-09-14T03:40:46.325 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.321+0000 7faf5e9e7640 1 -- 172.21.15.83:0/1992837672 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7faf58131610 con 0x7faf58094f10 2024-09-14T03:40:46.325 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.321+0000 7faf54ff9640 1 -- 172.21.15.83:0/1992837672 <== mon.2 v2:172.21.15.83:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7faf48035030 con 0x7faf58094f10 2024-09-14T03:40:46.325 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.321+0000 7faf5e9e7640 1 -- 172.21.15.83:0/1992837672 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7faf581238d0 con 0x7faf58094f10 2024-09-14T03:40:46.325 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.322+0000 7faf54ff9640 1 -- 172.21.15.83:0/1992837672 <== mon.2 v2:172.21.15.83:3302/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7faf48042070 con 0x7faf58094f10 2024-09-14T03:40:46.326 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.322+0000 7faf5e9e7640 1 -- 172.21.15.83:0/1992837672 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7faf580949d0 con 0x7faf58094f10 2024-09-14T03:40:46.326 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.322+0000 7faf54ff9640 1 -- 172.21.15.83:0/1992837672 <== mon.2 v2:172.21.15.83:3302/0 5 ==== osd_map(33..33 src has 1..33) ==== 3027+0+0 (secure 0 0 0) 0x7faf48089f70 con 0x7faf58094f10 2024-09-14T03:40:46.327 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.326+0000 7faf54ff9640 1 -- 172.21.15.83:0/1992837672 <== mon.2 v2:172.21.15.83:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+157506 (secure 0 0 0) 0x7faf48089020 con 0x7faf58094f10 2024-09-14T03:40:46.470 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.469+0000 7faf5e9e7640 1 -- 172.21.15.83:0/1992837672 --> v1:172.21.15.83:6800/1803897559 -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) -- 0x7faf580699c0 con 0x7faf440514e0 2024-09-14T03:40:46.471 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.470+0000 7faf54ff9640 1 -- 172.21.15.83:0/1992837672 <== mgr.14106 v1:172.21.15.83:6800/1803897559 1 ==== mgr_command_reply(tid 0: 0 dumped all) ==== 18+0+32015 (unknown 2965378022 0 3297738123) 0x7faf580699c0 con 0x7faf440514e0 2024-09-14T03:40:46.471 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:40:46.472 INFO:teuthology.orchestra.run.smithi083.stderr:dumped all 2024-09-14T03:40:46.473 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.471+0000 7faf5e9e7640 1 -- 172.21.15.83:0/1992837672 >> v1:172.21.15.83:6800/1803897559 conn(0x7faf440514e0 legacy=0x7faf440539a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:46.473 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.471+0000 7faf5e9e7640 1 -- 172.21.15.83:0/1992837672 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7faf58094f10 msgr2=0x7faf58130b90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:46.473 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.471+0000 7faf5e9e7640 1 --2- 172.21.15.83:0/1992837672 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7faf58094f10 0x7faf58130b90 secure :-1 s=READY pgs=63 cs=0 l=1 rev1=1 crypto rx=0x7faf48031000 tx=0x7faf480038a0 comp rx=0 tx=0).stop 2024-09-14T03:40:46.473 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.471+0000 7faf5e9e7640 1 -- 172.21.15.83:0/1992837672 shutdown_connections 2024-09-14T03:40:46.473 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.471+0000 7faf5e9e7640 1 --2- 172.21.15.83:0/1992837672 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7faf5812cc30 0x7faf581310d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:46.474 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.471+0000 7faf5e9e7640 1 --2- 172.21.15.83:0/1992837672 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7faf58094f10 0x7faf58130b90 unknown :-1 s=CLOSED pgs=63 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:46.474 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.471+0000 7faf5e9e7640 1 --2- 172.21.15.83:0/1992837672 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7faf580945d0 0x7faf58130650 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:46.474 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.471+0000 7faf5e9e7640 1 -- 172.21.15.83:0/1992837672 >> 172.21.15.83:0/1992837672 conn(0x7faf5809a2e0 msgr2=0x7faf5809afd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:46.474 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.472+0000 7faf5e9e7640 1 -- 172.21.15.83:0/1992837672 shutdown_connections 2024-09-14T03:40:46.474 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.472+0000 7faf5e9e7640 1 -- 172.21.15.83:0/1992837672 wait complete. 2024-09-14T03:40:46.484 INFO:teuthology.orchestra.run.smithi083.stdout:{"pg_ready":true,"pg_map":{"version":28,"stamp":"2024-09-14T03:40:46.043714+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":166,"num_read_kb":267,"num_write":151,"num_write_kb":1896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":107,"ondisk_log_size":107,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":4,"num_per_pool_osds":4,"num_per_pool_omap_osds":4,"kb":377487360,"kb_used":112364,"kb_used_data":2256,"kb_used_omap":17,"kb_used_meta":109998,"kb_avail":377374996,"statfs":{"total":386547056640,"available":386431995904,"internally_reserved":0,"allocated":2310144,"data_stored":1564350,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":17731,"internal_metadata":112638653},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.001397"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":54,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-14T03:40:41.119404+0000","last_change":"2024-09-14T03:40:35.310297+0000","last_active":"2024-09-14T03:40:41.119404+0000","last_peered":"2024-09-14T03:40:41.119404+0000","last_clean":"2024-09-14T03:40:41.119404+0000","last_became_active":"2024-09-14T03:40:35.309980+0000","last_became_peered":"2024-09-14T03:40:35.309980+0000","last_unstale":"2024-09-14T03:40:41.119404+0000","last_undegraded":"2024-09-14T03:40:41.119404+0000","last_fullsized":"2024-09-14T03:40:41.119404+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_clean_scrub_stamp":"2024-09-14T03:39:00.791161+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:33:56.556777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00098140899999999993,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":44,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-14T03:40:41.120097+0000","last_change":"2024-09-14T03:40:33.102548+0000","last_active":"2024-09-14T03:40:41.120097+0000","last_peered":"2024-09-14T03:40:41.120097+0000","last_clean":"2024-09-14T03:40:41.120097+0000","last_became_active":"2024-09-14T03:40:33.102279+0000","last_became_peered":"2024-09-14T03:40:33.102279+0000","last_unstale":"2024-09-14T03:40:41.120097+0000","last_undegraded":"2024-09-14T03:40:41.120097+0000","last_fullsized":"2024-09-14T03:40:41.120097+0000","mapping_epoch":27,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":28,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_clean_scrub_stamp":"2024-09-14T03:39:00.791161+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:26:35.820454+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00045411099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":54,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-14T03:40:41.119610+0000","last_change":"2024-09-14T03:40:35.309898+0000","last_active":"2024-09-14T03:40:41.119610+0000","last_peered":"2024-09-14T03:40:41.119610+0000","last_clean":"2024-09-14T03:40:41.119610+0000","last_became_active":"2024-09-14T03:40:35.309659+0000","last_became_peered":"2024-09-14T03:40:35.309659+0000","last_unstale":"2024-09-14T03:40:41.119610+0000","last_undegraded":"2024-09-14T03:40:41.119610+0000","last_fullsized":"2024-09-14T03:40:41.119610+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_clean_scrub_stamp":"2024-09-14T03:39:00.791161+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:46:05.387098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00087291699999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":44,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-14T03:40:41.120130+0000","last_change":"2024-09-14T03:40:33.102548+0000","last_active":"2024-09-14T03:40:41.120130+0000","last_peered":"2024-09-14T03:40:41.120130+0000","last_clean":"2024-09-14T03:40:41.120130+0000","last_became_active":"2024-09-14T03:40:33.102255+0000","last_became_peered":"2024-09-14T03:40:33.102255+0000","last_unstale":"2024-09-14T03:40:41.120130+0000","last_undegraded":"2024-09-14T03:40:41.120130+0000","last_fullsized":"2024-09-14T03:40:41.120130+0000","mapping_epoch":27,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":28,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_clean_scrub_stamp":"2024-09-14T03:39:00.791161+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:46:51.929694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00048052199999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"18'2","reported_seq":52,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-14T03:40:41.119384+0000","last_change":"2024-09-14T03:40:35.110955+0000","last_active":"2024-09-14T03:40:41.119384+0000","last_peered":"2024-09-14T03:40:41.119384+0000","last_clean":"2024-09-14T03:40:41.119384+0000","last_became_active":"2024-09-14T03:40:35.110637+0000","last_became_peered":"2024-09-14T03:40:35.110637+0000","last_unstale":"2024-09-14T03:40:41.119384+0000","last_undegraded":"2024-09-14T03:40:41.119384+0000","last_fullsized":"2024-09-14T03:40:41.119384+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_clean_scrub_stamp":"2024-09-14T03:39:00.791161+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:35:22.417821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00135844,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":37,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-14T03:40:41.177732+0000","last_change":"2024-09-14T03:40:36.107870+0000","last_active":"2024-09-14T03:40:41.177732+0000","last_peered":"2024-09-14T03:40:41.177732+0000","last_clean":"2024-09-14T03:40:41.177732+0000","last_became_active":"2024-09-14T03:40:36.107387+0000","last_became_peered":"2024-09-14T03:40:36.107387+0000","last_unstale":"2024-09-14T03:40:41.177732+0000","last_undegraded":"2024-09-14T03:40:41.177732+0000","last_fullsized":"2024-09-14T03:40:41.177732+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_clean_scrub_stamp":"2024-09-14T03:39:00.791161+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:24:43.135910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046880499999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":50,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-14T03:40:41.119371+0000","last_change":"2024-09-14T03:40:35.105432+0000","last_active":"2024-09-14T03:40:41.119371+0000","last_peered":"2024-09-14T03:40:41.119371+0000","last_clean":"2024-09-14T03:40:41.119371+0000","last_became_active":"2024-09-14T03:40:35.105125+0000","last_became_peered":"2024-09-14T03:40:35.105125+0000","last_unstale":"2024-09-14T03:40:41.119371+0000","last_undegraded":"2024-09-14T03:40:41.119371+0000","last_fullsized":"2024-09-14T03:40:41.119371+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_clean_scrub_stamp":"2024-09-14T03:39:00.791161+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:06:08.387860+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000765802,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"16'1","reported_seq":48,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-14T03:40:41.119494+0000","last_change":"2024-09-14T03:40:36.107793+0000","last_active":"2024-09-14T03:40:41.119494+0000","last_peered":"2024-09-14T03:40:41.119494+0000","last_clean":"2024-09-14T03:40:41.119494+0000","last_became_active":"2024-09-14T03:40:36.107394+0000","last_became_peered":"2024-09-14T03:40:36.107394+0000","last_unstale":"2024-09-14T03:40:41.119494+0000","last_undegraded":"2024-09-14T03:40:41.119494+0000","last_fullsized":"2024-09-14T03:40:41.119494+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_clean_scrub_stamp":"2024-09-14T03:39:00.791161+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:34:57.278411+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00075472199999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"21'104","reported_seq":199,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-14T03:40:41.119989+0000","last_change":"2024-09-14T03:40:33.097410+0000","last_active":"2024-09-14T03:40:41.119989+0000","last_peered":"2024-09-14T03:40:41.119989+0000","last_clean":"2024-09-14T03:40:41.119989+0000","last_became_active":"2024-09-14T03:40:33.097097+0000","last_became_peered":"2024-09-14T03:40:33.097097+0000","last_unstale":"2024-09-14T03:40:41.119989+0000","last_undegraded":"2024-09-14T03:40:41.119989+0000","last_fullsized":"2024-09-14T03:40:41.119989+0000","mapping_epoch":27,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":28,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:38:57.773685+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:57.773685+0000","last_clean_scrub_stamp":"2024-09-14T03:38:57.773685+0000","objects_scrubbed":0,"log_size":104,"log_dups_size":0,"ondisk_log_size":104,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:08:55.427294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":166,"num_read_kb":267,"num_write":149,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":4},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":166,"num_read_kb":267,"num_write":149,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":104,"ondisk_log_size":104,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":3,"up_from":29,"seq":124554051588,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27820,"kb_used_data":276,"kb_used_omap":4,"kb_used_meta":27515,"kb_avail":94344020,"statfs":{"total":96636764160,"available":96608276480,"internally_reserved":0,"allocated":282624,"data_stored":95913,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4433,"internal_metadata":28176047},"hb_peers":[0,1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":30,"seq":128849018884,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27812,"kb_used_data":272,"kb_used_omap":4,"kb_used_meta":27515,"kb_avail":94344028,"statfs":{"total":96636764160,"available":96608284672,"internally_reserved":0,"allocated":278528,"data_stored":95894,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4432,"internal_metadata":28176048},"hb_peers":[0,1,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":27,"seq":115964116996,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":28336,"kb_used_data":856,"kb_used_omap":4,"kb_used_meta":27451,"kb_avail":94343504,"statfs":{"total":96636764160,"available":96607748096,"internally_reserved":0,"allocated":876544,"data_stored":686281,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4431,"internal_metadata":28110513},"hb_peers":[0,2,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":27,"seq":115964116996,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":28396,"kb_used_data":852,"kb_used_omap":4,"kb_used_meta":27515,"kb_avail":94343444,"statfs":{"total":96636764160,"available":96607686656,"internally_reserved":0,"allocated":872448,"data_stored":686262,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4435,"internal_metadata":28176045},"hb_peers":[1,2,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T03:40:46.485 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-09-14T03:40:46.585 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.583+0000 7fce5f1be640 1 -- 172.21.15.83:0/4150695764 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fce5812bdb0 msgr2=0x7fce580945d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:46.585 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.583+0000 7fce5f1be640 1 --2- 172.21.15.83:0/4150695764 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fce5812bdb0 0x7fce580945d0 secure :-1 s=READY pgs=116 cs=0 l=1 rev1=1 crypto rx=0x7fce44009830 tx=0x7fce4402a3f0 comp rx=0 tx=0).stop 2024-09-14T03:40:46.585 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.583+0000 7fce5f1be640 1 -- 172.21.15.83:0/4150695764 shutdown_connections 2024-09-14T03:40:46.585 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.583+0000 7fce5f1be640 1 --2- 172.21.15.83:0/4150695764 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fce5812bdb0 0x7fce580945d0 unknown :-1 s=CLOSED pgs=116 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:46.585 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.583+0000 7fce5f1be640 1 --2- 172.21.15.83:0/4150695764 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fce5812b490 0x7fce5812b870 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:46.585 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.583+0000 7fce5f1be640 1 -- 172.21.15.83:0/4150695764 >> 172.21.15.83:0/4150695764 conn(0x7fce5809a2e0 msgr2=0x7fce58097ad0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:46.586 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.583+0000 7fce5f1be640 1 -- 172.21.15.83:0/4150695764 shutdown_connections 2024-09-14T03:40:46.586 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.583+0000 7fce5f1be640 1 -- 172.21.15.83:0/4150695764 wait complete. 2024-09-14T03:40:46.586 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.584+0000 7fce5f1be640 1 Processor -- start 2024-09-14T03:40:46.586 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.584+0000 7fce5f1be640 1 -- start start 2024-09-14T03:40:46.588 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.584+0000 7fce5f1be640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fce58094b10 0x7fce581c0690 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:46.588 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.584+0000 7fce5f1be640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fce5812b490 0x7fce581c0bd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:46.588 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.584+0000 7fce5f1be640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fce5812bdb0 0x7fce581c1110 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:46.588 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.584+0000 7fce5f1be640 1 -- --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7fce58133dd0 con 0x7fce5812b490 2024-09-14T03:40:46.588 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.584+0000 7fce5f1be640 1 -- --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7fce58133c50 con 0x7fce58094b10 2024-09-14T03:40:46.588 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.584+0000 7fce5f1be640 1 -- --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7fce58133f50 con 0x7fce5812bdb0 2024-09-14T03:40:46.588 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.584+0000 7fce57fff640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fce5812b490 0x7fce581c0bd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:46.588 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.584+0000 7fce5cf5a640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fce58094b10 0x7fce581c0690 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:46.589 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.584+0000 7fce57fff640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fce5812b490 0x7fce581c0bd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3300/0 says I am v2:172.21.15.83:40144/0 (socket says 172.21.15.83:40144) 2024-09-14T03:40:46.589 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.584+0000 7fce57fff640 1 -- 172.21.15.83:0/1148316162 learned_addr learned my addr 172.21.15.83:0/1148316162 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:40:46.589 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.584+0000 7fce5cf5a640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fce58094b10 0x7fce581c0690 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3301/0 says I am v2:172.21.15.83:53858/0 (socket says 172.21.15.83:53858) 2024-09-14T03:40:46.589 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.584+0000 7fce5d75b640 1 --2- 172.21.15.83:0/1148316162 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fce5812bdb0 0x7fce581c1110 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:46.589 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.585+0000 7fce5cf5a640 1 -- 172.21.15.83:0/1148316162 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fce5812bdb0 msgr2=0x7fce581c1110 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:46.589 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.585+0000 7fce5cf5a640 1 --2- 172.21.15.83:0/1148316162 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fce5812bdb0 0x7fce581c1110 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:46.589 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.585+0000 7fce5cf5a640 1 -- 172.21.15.83:0/1148316162 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fce5812b490 msgr2=0x7fce581c0bd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:46.589 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.585+0000 7fce5cf5a640 1 --2- 172.21.15.83:0/1148316162 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fce5812b490 0x7fce581c0bd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:46.589 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.585+0000 7fce5cf5a640 1 -- 172.21.15.83:0/1148316162 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fce58074030 con 0x7fce58094b10 2024-09-14T03:40:46.589 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.585+0000 7fce57fff640 1 --2- 172.21.15.83:0/1148316162 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fce5812b490 0x7fce581c0bd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:40:46.589 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.585+0000 7fce5d75b640 1 --2- 172.21.15.83:0/1148316162 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fce5812bdb0 0x7fce581c1110 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:40:46.589 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.585+0000 7fce5cf5a640 1 --2- 172.21.15.83:0/1148316162 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fce58094b10 0x7fce581c0690 secure :-1 s=READY pgs=64 cs=0 l=1 rev1=1 crypto rx=0x7fce4c005fd0 tx=0x7fce4c00ee50 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:40:46.589 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.585+0000 7fce557fa640 1 -- 172.21.15.83:0/1148316162 <== mon.1 v2:172.21.15.83:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fce4c00cd00 con 0x7fce58094b10 2024-09-14T03:40:46.590 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.585+0000 7fce557fa640 1 -- 172.21.15.83:0/1148316162 <== mon.1 v2:172.21.15.83:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fce4c004020 con 0x7fce58094b10 2024-09-14T03:40:46.590 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.585+0000 7fce5f1be640 1 -- 172.21.15.83:0/1148316162 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fce5812ee00 con 0x7fce58094b10 2024-09-14T03:40:46.590 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.585+0000 7fce557fa640 1 -- 172.21.15.83:0/1148316162 <== mon.1 v2:172.21.15.83:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fce4c004410 con 0x7fce58094b10 2024-09-14T03:40:46.590 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.585+0000 7fce5f1be640 1 -- 172.21.15.83:0/1148316162 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7fce5812a210 con 0x7fce58094b10 2024-09-14T03:40:46.590 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.586+0000 7fce5f1be640 1 -- 172.21.15.83:0/1148316162 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fce5812b870 con 0x7fce58094b10 2024-09-14T03:40:46.590 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.586+0000 7fce557fa640 1 -- 172.21.15.83:0/1148316162 <== mon.1 v2:172.21.15.83:3301/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7fce4c0035f0 con 0x7fce58094b10 2024-09-14T03:40:46.590 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.587+0000 7fce557fa640 1 -- 172.21.15.83:0/1148316162 <== mon.1 v2:172.21.15.83:3301/0 5 ==== osd_map(33..33 src has 1..33) ==== 3027+0+0 (secure 0 0 0) 0x7fce4c0698a0 con 0x7fce58094b10 2024-09-14T03:40:46.591 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.590+0000 7fce557fa640 1 -- 172.21.15.83:0/1148316162 <== mon.1 v2:172.21.15.83:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+157506 (secure 0 0 0) 0x7fce4c068950 con 0x7fce58094b10 2024-09-14T03:40:46.736 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.735+0000 7fce5f1be640 1 -- 172.21.15.83:0/1148316162 --> v1:172.21.15.83:6800/1803897559 -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) -- 0x7fce5812dde0 con 0x7fce3c0514e0 2024-09-14T03:40:46.737 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.736+0000 7fce557fa640 1 -- 172.21.15.83:0/1148316162 <== mgr.14106 v1:172.21.15.83:6800/1803897559 1 ==== mgr_command_reply(tid 0: 0 dumped all) ==== 18+0+32015 (unknown 2965378022 0 3297738123) 0x7fce5812dde0 con 0x7fce3c0514e0 2024-09-14T03:40:46.738 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:40:46.738 INFO:teuthology.orchestra.run.smithi083.stderr:dumped all 2024-09-14T03:40:46.740 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.737+0000 7fce5f1be640 1 -- 172.21.15.83:0/1148316162 >> v1:172.21.15.83:6800/1803897559 conn(0x7fce3c0514e0 legacy=0x7fce3c0539a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:46.740 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.737+0000 7fce5f1be640 1 -- 172.21.15.83:0/1148316162 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fce58094b10 msgr2=0x7fce581c0690 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:46.740 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.737+0000 7fce5f1be640 1 --2- 172.21.15.83:0/1148316162 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fce58094b10 0x7fce581c0690 secure :-1 s=READY pgs=64 cs=0 l=1 rev1=1 crypto rx=0x7fce4c005fd0 tx=0x7fce4c00ee50 comp rx=0 tx=0).stop 2024-09-14T03:40:46.741 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.738+0000 7fce5f1be640 1 -- 172.21.15.83:0/1148316162 shutdown_connections 2024-09-14T03:40:46.741 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.738+0000 7fce5f1be640 1 --2- 172.21.15.83:0/1148316162 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fce5812bdb0 0x7fce581c1110 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:46.741 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.738+0000 7fce5f1be640 1 --2- 172.21.15.83:0/1148316162 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fce5812b490 0x7fce581c0bd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:46.741 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.738+0000 7fce5f1be640 1 --2- 172.21.15.83:0/1148316162 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fce58094b10 0x7fce581c0690 unknown :-1 s=CLOSED pgs=64 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:46.741 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.738+0000 7fce5f1be640 1 -- 172.21.15.83:0/1148316162 >> 172.21.15.83:0/1148316162 conn(0x7fce5809a2e0 msgr2=0x7fce580901a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:46.741 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.738+0000 7fce5f1be640 1 -- 172.21.15.83:0/1148316162 shutdown_connections 2024-09-14T03:40:46.741 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.738+0000 7fce5f1be640 1 -- 172.21.15.83:0/1148316162 wait complete. 2024-09-14T03:40:46.750 INFO:teuthology.orchestra.run.smithi083.stdout:{"pg_ready":true,"pg_map":{"version":28,"stamp":"2024-09-14T03:40:46.043714+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":166,"num_read_kb":267,"num_write":151,"num_write_kb":1896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":107,"ondisk_log_size":107,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":4,"num_per_pool_osds":4,"num_per_pool_omap_osds":4,"kb":377487360,"kb_used":112364,"kb_used_data":2256,"kb_used_omap":17,"kb_used_meta":109998,"kb_avail":377374996,"statfs":{"total":386547056640,"available":386431995904,"internally_reserved":0,"allocated":2310144,"data_stored":1564350,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":17731,"internal_metadata":112638653},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.001397"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":54,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-14T03:40:41.119404+0000","last_change":"2024-09-14T03:40:35.310297+0000","last_active":"2024-09-14T03:40:41.119404+0000","last_peered":"2024-09-14T03:40:41.119404+0000","last_clean":"2024-09-14T03:40:41.119404+0000","last_became_active":"2024-09-14T03:40:35.309980+0000","last_became_peered":"2024-09-14T03:40:35.309980+0000","last_unstale":"2024-09-14T03:40:41.119404+0000","last_undegraded":"2024-09-14T03:40:41.119404+0000","last_fullsized":"2024-09-14T03:40:41.119404+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_clean_scrub_stamp":"2024-09-14T03:39:00.791161+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:33:56.556777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00098140899999999993,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":44,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-14T03:40:41.120097+0000","last_change":"2024-09-14T03:40:33.102548+0000","last_active":"2024-09-14T03:40:41.120097+0000","last_peered":"2024-09-14T03:40:41.120097+0000","last_clean":"2024-09-14T03:40:41.120097+0000","last_became_active":"2024-09-14T03:40:33.102279+0000","last_became_peered":"2024-09-14T03:40:33.102279+0000","last_unstale":"2024-09-14T03:40:41.120097+0000","last_undegraded":"2024-09-14T03:40:41.120097+0000","last_fullsized":"2024-09-14T03:40:41.120097+0000","mapping_epoch":27,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":28,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_clean_scrub_stamp":"2024-09-14T03:39:00.791161+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:26:35.820454+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00045411099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":54,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-14T03:40:41.119610+0000","last_change":"2024-09-14T03:40:35.309898+0000","last_active":"2024-09-14T03:40:41.119610+0000","last_peered":"2024-09-14T03:40:41.119610+0000","last_clean":"2024-09-14T03:40:41.119610+0000","last_became_active":"2024-09-14T03:40:35.309659+0000","last_became_peered":"2024-09-14T03:40:35.309659+0000","last_unstale":"2024-09-14T03:40:41.119610+0000","last_undegraded":"2024-09-14T03:40:41.119610+0000","last_fullsized":"2024-09-14T03:40:41.119610+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_clean_scrub_stamp":"2024-09-14T03:39:00.791161+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:46:05.387098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00087291699999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":44,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-14T03:40:41.120130+0000","last_change":"2024-09-14T03:40:33.102548+0000","last_active":"2024-09-14T03:40:41.120130+0000","last_peered":"2024-09-14T03:40:41.120130+0000","last_clean":"2024-09-14T03:40:41.120130+0000","last_became_active":"2024-09-14T03:40:33.102255+0000","last_became_peered":"2024-09-14T03:40:33.102255+0000","last_unstale":"2024-09-14T03:40:41.120130+0000","last_undegraded":"2024-09-14T03:40:41.120130+0000","last_fullsized":"2024-09-14T03:40:41.120130+0000","mapping_epoch":27,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":28,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_clean_scrub_stamp":"2024-09-14T03:39:00.791161+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:46:51.929694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00048052199999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"18'2","reported_seq":52,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-14T03:40:41.119384+0000","last_change":"2024-09-14T03:40:35.110955+0000","last_active":"2024-09-14T03:40:41.119384+0000","last_peered":"2024-09-14T03:40:41.119384+0000","last_clean":"2024-09-14T03:40:41.119384+0000","last_became_active":"2024-09-14T03:40:35.110637+0000","last_became_peered":"2024-09-14T03:40:35.110637+0000","last_unstale":"2024-09-14T03:40:41.119384+0000","last_undegraded":"2024-09-14T03:40:41.119384+0000","last_fullsized":"2024-09-14T03:40:41.119384+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_clean_scrub_stamp":"2024-09-14T03:39:00.791161+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:35:22.417821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00135844,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":37,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-14T03:40:41.177732+0000","last_change":"2024-09-14T03:40:36.107870+0000","last_active":"2024-09-14T03:40:41.177732+0000","last_peered":"2024-09-14T03:40:41.177732+0000","last_clean":"2024-09-14T03:40:41.177732+0000","last_became_active":"2024-09-14T03:40:36.107387+0000","last_became_peered":"2024-09-14T03:40:36.107387+0000","last_unstale":"2024-09-14T03:40:41.177732+0000","last_undegraded":"2024-09-14T03:40:41.177732+0000","last_fullsized":"2024-09-14T03:40:41.177732+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_clean_scrub_stamp":"2024-09-14T03:39:00.791161+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:24:43.135910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046880499999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":50,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-14T03:40:41.119371+0000","last_change":"2024-09-14T03:40:35.105432+0000","last_active":"2024-09-14T03:40:41.119371+0000","last_peered":"2024-09-14T03:40:41.119371+0000","last_clean":"2024-09-14T03:40:41.119371+0000","last_became_active":"2024-09-14T03:40:35.105125+0000","last_became_peered":"2024-09-14T03:40:35.105125+0000","last_unstale":"2024-09-14T03:40:41.119371+0000","last_undegraded":"2024-09-14T03:40:41.119371+0000","last_fullsized":"2024-09-14T03:40:41.119371+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_clean_scrub_stamp":"2024-09-14T03:39:00.791161+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:06:08.387860+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000765802,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"16'1","reported_seq":48,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-14T03:40:41.119494+0000","last_change":"2024-09-14T03:40:36.107793+0000","last_active":"2024-09-14T03:40:41.119494+0000","last_peered":"2024-09-14T03:40:41.119494+0000","last_clean":"2024-09-14T03:40:41.119494+0000","last_became_active":"2024-09-14T03:40:36.107394+0000","last_became_peered":"2024-09-14T03:40:36.107394+0000","last_unstale":"2024-09-14T03:40:41.119494+0000","last_undegraded":"2024-09-14T03:40:41.119494+0000","last_fullsized":"2024-09-14T03:40:41.119494+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_clean_scrub_stamp":"2024-09-14T03:39:00.791161+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:34:57.278411+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00075472199999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"21'104","reported_seq":199,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-14T03:40:41.119989+0000","last_change":"2024-09-14T03:40:33.097410+0000","last_active":"2024-09-14T03:40:41.119989+0000","last_peered":"2024-09-14T03:40:41.119989+0000","last_clean":"2024-09-14T03:40:41.119989+0000","last_became_active":"2024-09-14T03:40:33.097097+0000","last_became_peered":"2024-09-14T03:40:33.097097+0000","last_unstale":"2024-09-14T03:40:41.119989+0000","last_undegraded":"2024-09-14T03:40:41.119989+0000","last_fullsized":"2024-09-14T03:40:41.119989+0000","mapping_epoch":27,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":28,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:38:57.773685+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:57.773685+0000","last_clean_scrub_stamp":"2024-09-14T03:38:57.773685+0000","objects_scrubbed":0,"log_size":104,"log_dups_size":0,"ondisk_log_size":104,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:08:55.427294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":166,"num_read_kb":267,"num_write":149,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":4},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":166,"num_read_kb":267,"num_write":149,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":104,"ondisk_log_size":104,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":3,"up_from":29,"seq":124554051588,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27820,"kb_used_data":276,"kb_used_omap":4,"kb_used_meta":27515,"kb_avail":94344020,"statfs":{"total":96636764160,"available":96608276480,"internally_reserved":0,"allocated":282624,"data_stored":95913,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4433,"internal_metadata":28176047},"hb_peers":[0,1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":30,"seq":128849018884,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27812,"kb_used_data":272,"kb_used_omap":4,"kb_used_meta":27515,"kb_avail":94344028,"statfs":{"total":96636764160,"available":96608284672,"internally_reserved":0,"allocated":278528,"data_stored":95894,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4432,"internal_metadata":28176048},"hb_peers":[0,1,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":27,"seq":115964116996,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":28336,"kb_used_data":856,"kb_used_omap":4,"kb_used_meta":27451,"kb_avail":94343504,"statfs":{"total":96636764160,"available":96607748096,"internally_reserved":0,"allocated":876544,"data_stored":686281,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4431,"internal_metadata":28110513},"hb_peers":[0,2,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":27,"seq":115964116996,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":28396,"kb_used_data":852,"kb_used_omap":4,"kb_used_meta":27515,"kb_avail":94343444,"statfs":{"total":96636764160,"available":96607686656,"internally_reserved":0,"allocated":872448,"data_stored":686262,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4435,"internal_metadata":28176045},"hb_peers":[1,2,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T03:40:46.751 INFO:tasks.ceph.ceph_manager.ceph:clean! 2024-09-14T03:40:46.751 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-09-14T03:40:46.849 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.847+0000 7fd1985c9640 1 -- 172.21.15.83:0/2548582310 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fd1901313c0 msgr2=0x7fd1901337b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:46.850 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.847+0000 7fd1985c9640 1 --2- 172.21.15.83:0/2548582310 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fd1901313c0 0x7fd1901337b0 secure :-1 s=READY pgs=118 cs=0 l=1 rev1=1 crypto rx=0x7fd188005fd0 tx=0x7fd18802a780 comp rx=0 tx=0).stop 2024-09-14T03:40:46.850 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.848+0000 7fd1985c9640 1 -- 172.21.15.83:0/2548582310 shutdown_connections 2024-09-14T03:40:46.850 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.848+0000 7fd1985c9640 1 --2- 172.21.15.83:0/2548582310 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fd1901313c0 0x7fd1901337b0 unknown :-1 s=CLOSED pgs=118 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:46.850 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.848+0000 7fd1985c9640 1 --2- 172.21.15.83:0/2548582310 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fd19012ea90 0x7fd190130e80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:46.850 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.848+0000 7fd1985c9640 1 -- 172.21.15.83:0/2548582310 >> 172.21.15.83:0/2548582310 conn(0x7fd19009c230 msgr2=0x7fd19009c640 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:46.850 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.848+0000 7fd1985c9640 1 -- 172.21.15.83:0/2548582310 shutdown_connections 2024-09-14T03:40:46.850 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.848+0000 7fd1985c9640 1 -- 172.21.15.83:0/2548582310 wait complete. 2024-09-14T03:40:46.852 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.848+0000 7fd1985c9640 1 Processor -- start 2024-09-14T03:40:46.852 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.848+0000 7fd1985c9640 1 -- start start 2024-09-14T03:40:46.852 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.849+0000 7fd1985c9640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fd190095c60 0x7fd19006f880 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:46.852 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.849+0000 7fd1985c9640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fd19012ea90 0x7fd19006fdc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:46.852 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.849+0000 7fd1985c9640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fd1901313c0 0x7fd190070300 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:46.852 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.849+0000 7fd1985c9640 1 -- --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7fd190134720 con 0x7fd190095c60 2024-09-14T03:40:46.852 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.849+0000 7fd1985c9640 1 -- --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7fd1901345a0 con 0x7fd1901313c0 2024-09-14T03:40:46.853 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.849+0000 7fd1985c9640 1 -- --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7fd1901348a0 con 0x7fd19012ea90 2024-09-14T03:40:46.853 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.849+0000 7fd196b66640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fd1901313c0 0x7fd190070300 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:46.853 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.849+0000 7fd196365640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fd190095c60 0x7fd19006f880 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:46.853 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.849+0000 7fd195b64640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fd19012ea90 0x7fd19006fdc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:46.853 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.849+0000 7fd196365640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fd190095c60 0x7fd19006f880 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3300/0 says I am v2:172.21.15.83:40152/0 (socket says 172.21.15.83:40152) 2024-09-14T03:40:46.853 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.849+0000 7fd196b66640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fd1901313c0 0x7fd190070300 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3301/0 says I am v2:172.21.15.83:53872/0 (socket says 172.21.15.83:53872) 2024-09-14T03:40:46.853 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.849+0000 7fd196365640 1 -- 172.21.15.83:0/378931034 learned_addr learned my addr 172.21.15.83:0/378931034 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:40:46.853 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.849+0000 7fd196365640 1 -- 172.21.15.83:0/378931034 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fd19012ea90 msgr2=0x7fd19006fdc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:46.853 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.849+0000 7fd196365640 1 --2- 172.21.15.83:0/378931034 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fd19012ea90 0x7fd19006fdc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:46.853 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.849+0000 7fd196365640 1 -- 172.21.15.83:0/378931034 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fd1901313c0 msgr2=0x7fd190070300 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:46.853 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.849+0000 7fd196365640 1 --2- 172.21.15.83:0/378931034 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fd1901313c0 0x7fd190070300 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:46.853 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.849+0000 7fd196365640 1 -- 172.21.15.83:0/378931034 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fd190096f60 con 0x7fd190095c60 2024-09-14T03:40:46.853 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.849+0000 7fd196b66640 1 --2- 172.21.15.83:0/378931034 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fd1901313c0 0x7fd190070300 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:40:46.854 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.849+0000 7fd196365640 1 --2- 172.21.15.83:0/378931034 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fd190095c60 0x7fd19006f880 secure :-1 s=READY pgs=119 cs=0 l=1 rev1=1 crypto rx=0x7fd18c00c410 tx=0x7fd18c00c8d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:40:46.854 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.850+0000 7fd186ffd640 1 -- 172.21.15.83:0/378931034 <== mon.0 v2:172.21.15.83:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd18c017070 con 0x7fd190095c60 2024-09-14T03:40:46.854 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.850+0000 7fd186ffd640 1 -- 172.21.15.83:0/378931034 <== mon.0 v2:172.21.15.83:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fd18c0035f0 con 0x7fd190095c60 2024-09-14T03:40:46.854 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.850+0000 7fd186ffd640 1 -- 172.21.15.83:0/378931034 <== mon.0 v2:172.21.15.83:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd18c003a40 con 0x7fd190095c60 2024-09-14T03:40:46.854 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.850+0000 7fd1985c9640 1 -- 172.21.15.83:0/378931034 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fd190070a90 con 0x7fd190095c60 2024-09-14T03:40:46.854 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.850+0000 7fd1985c9640 1 -- 172.21.15.83:0/378931034 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fd19012c0b0 con 0x7fd190095c60 2024-09-14T03:40:46.854 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.851+0000 7fd1985c9640 1 -- 172.21.15.83:0/378931034 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fd19012c310 con 0x7fd190095c60 2024-09-14T03:40:46.854 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.851+0000 7fd186ffd640 1 -- 172.21.15.83:0/378931034 <== mon.0 v2:172.21.15.83:3300/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7fd18c01a070 con 0x7fd190095c60 2024-09-14T03:40:46.854 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.851+0000 7fd186ffd640 1 -- 172.21.15.83:0/378931034 <== mon.0 v2:172.21.15.83:3300/0 5 ==== osd_map(33..33 src has 1..33) ==== 3027+0+0 (secure 0 0 0) 0x7fd18c068390 con 0x7fd190095c60 2024-09-14T03:40:46.856 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.855+0000 7fd186ffd640 1 -- 172.21.15.83:0/378931034 <== mon.0 v2:172.21.15.83:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+157506 (secure 0 0 0) 0x7fd18c067440 con 0x7fd190095c60 2024-09-14T03:40:46.998 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.997+0000 7fd1985c9640 1 -- 172.21.15.83:0/378931034 --> v1:172.21.15.83:6800/1803897559 -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) -- 0x7fd190096040 con 0x7fd1740514b0 2024-09-14T03:40:46.999 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.997+0000 7fd186ffd640 1 -- 172.21.15.83:0/378931034 <== mgr.14106 v1:172.21.15.83:6800/1803897559 1 ==== mgr_command_reply(tid 0: 0 dumped all) ==== 18+0+32015 (unknown 2965378022 0 3297738123) 0x7fd190096040 con 0x7fd1740514b0 2024-09-14T03:40:46.999 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:40:47.000 INFO:teuthology.orchestra.run.smithi083.stderr:dumped all 2024-09-14T03:40:47.001 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.999+0000 7fd1985c9640 1 -- 172.21.15.83:0/378931034 >> v1:172.21.15.83:6800/1803897559 conn(0x7fd1740514b0 legacy=0x7fd174053970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:47.001 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.999+0000 7fd1985c9640 1 -- 172.21.15.83:0/378931034 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fd190095c60 msgr2=0x7fd19006f880 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:47.001 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.999+0000 7fd1985c9640 1 --2- 172.21.15.83:0/378931034 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fd190095c60 0x7fd19006f880 secure :-1 s=READY pgs=119 cs=0 l=1 rev1=1 crypto rx=0x7fd18c00c410 tx=0x7fd18c00c8d0 comp rx=0 tx=0).stop 2024-09-14T03:40:47.001 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.999+0000 7fd1985c9640 1 -- 172.21.15.83:0/378931034 shutdown_connections 2024-09-14T03:40:47.001 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.999+0000 7fd1985c9640 1 --2- 172.21.15.83:0/378931034 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fd1901313c0 0x7fd190070300 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:47.001 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.999+0000 7fd1985c9640 1 --2- 172.21.15.83:0/378931034 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fd19012ea90 0x7fd19006fdc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:47.001 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.999+0000 7fd1985c9640 1 --2- 172.21.15.83:0/378931034 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fd190095c60 0x7fd19006f880 unknown :-1 s=CLOSED pgs=119 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:47.002 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.999+0000 7fd1985c9640 1 -- 172.21.15.83:0/378931034 >> 172.21.15.83:0/378931034 conn(0x7fd19009c230 msgr2=0x7fd190099d00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:47.002 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.999+0000 7fd1985c9640 1 -- 172.21.15.83:0/378931034 shutdown_connections 2024-09-14T03:40:47.002 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:46.999+0000 7fd1985c9640 1 -- 172.21.15.83:0/378931034 wait complete. 2024-09-14T03:40:47.012 INFO:teuthology.orchestra.run.smithi083.stdout:{"pg_ready":true,"pg_map":{"version":28,"stamp":"2024-09-14T03:40:46.043714+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":166,"num_read_kb":267,"num_write":151,"num_write_kb":1896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":107,"ondisk_log_size":107,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":4,"num_per_pool_osds":4,"num_per_pool_omap_osds":4,"kb":377487360,"kb_used":112364,"kb_used_data":2256,"kb_used_omap":17,"kb_used_meta":109998,"kb_avail":377374996,"statfs":{"total":386547056640,"available":386431995904,"internally_reserved":0,"allocated":2310144,"data_stored":1564350,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":17731,"internal_metadata":112638653},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.001397"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":54,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-14T03:40:41.119404+0000","last_change":"2024-09-14T03:40:35.310297+0000","last_active":"2024-09-14T03:40:41.119404+0000","last_peered":"2024-09-14T03:40:41.119404+0000","last_clean":"2024-09-14T03:40:41.119404+0000","last_became_active":"2024-09-14T03:40:35.309980+0000","last_became_peered":"2024-09-14T03:40:35.309980+0000","last_unstale":"2024-09-14T03:40:41.119404+0000","last_undegraded":"2024-09-14T03:40:41.119404+0000","last_fullsized":"2024-09-14T03:40:41.119404+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_clean_scrub_stamp":"2024-09-14T03:39:00.791161+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:33:56.556777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00098140899999999993,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":44,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-14T03:40:41.120097+0000","last_change":"2024-09-14T03:40:33.102548+0000","last_active":"2024-09-14T03:40:41.120097+0000","last_peered":"2024-09-14T03:40:41.120097+0000","last_clean":"2024-09-14T03:40:41.120097+0000","last_became_active":"2024-09-14T03:40:33.102279+0000","last_became_peered":"2024-09-14T03:40:33.102279+0000","last_unstale":"2024-09-14T03:40:41.120097+0000","last_undegraded":"2024-09-14T03:40:41.120097+0000","last_fullsized":"2024-09-14T03:40:41.120097+0000","mapping_epoch":27,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":28,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_clean_scrub_stamp":"2024-09-14T03:39:00.791161+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:26:35.820454+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00045411099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":54,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-14T03:40:41.119610+0000","last_change":"2024-09-14T03:40:35.309898+0000","last_active":"2024-09-14T03:40:41.119610+0000","last_peered":"2024-09-14T03:40:41.119610+0000","last_clean":"2024-09-14T03:40:41.119610+0000","last_became_active":"2024-09-14T03:40:35.309659+0000","last_became_peered":"2024-09-14T03:40:35.309659+0000","last_unstale":"2024-09-14T03:40:41.119610+0000","last_undegraded":"2024-09-14T03:40:41.119610+0000","last_fullsized":"2024-09-14T03:40:41.119610+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_clean_scrub_stamp":"2024-09-14T03:39:00.791161+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:46:05.387098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00087291699999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":44,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-14T03:40:41.120130+0000","last_change":"2024-09-14T03:40:33.102548+0000","last_active":"2024-09-14T03:40:41.120130+0000","last_peered":"2024-09-14T03:40:41.120130+0000","last_clean":"2024-09-14T03:40:41.120130+0000","last_became_active":"2024-09-14T03:40:33.102255+0000","last_became_peered":"2024-09-14T03:40:33.102255+0000","last_unstale":"2024-09-14T03:40:41.120130+0000","last_undegraded":"2024-09-14T03:40:41.120130+0000","last_fullsized":"2024-09-14T03:40:41.120130+0000","mapping_epoch":27,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":28,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_clean_scrub_stamp":"2024-09-14T03:39:00.791161+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:46:51.929694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00048052199999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"18'2","reported_seq":52,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-14T03:40:41.119384+0000","last_change":"2024-09-14T03:40:35.110955+0000","last_active":"2024-09-14T03:40:41.119384+0000","last_peered":"2024-09-14T03:40:41.119384+0000","last_clean":"2024-09-14T03:40:41.119384+0000","last_became_active":"2024-09-14T03:40:35.110637+0000","last_became_peered":"2024-09-14T03:40:35.110637+0000","last_unstale":"2024-09-14T03:40:41.119384+0000","last_undegraded":"2024-09-14T03:40:41.119384+0000","last_fullsized":"2024-09-14T03:40:41.119384+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_clean_scrub_stamp":"2024-09-14T03:39:00.791161+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:35:22.417821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00135844,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":37,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-14T03:40:41.177732+0000","last_change":"2024-09-14T03:40:36.107870+0000","last_active":"2024-09-14T03:40:41.177732+0000","last_peered":"2024-09-14T03:40:41.177732+0000","last_clean":"2024-09-14T03:40:41.177732+0000","last_became_active":"2024-09-14T03:40:36.107387+0000","last_became_peered":"2024-09-14T03:40:36.107387+0000","last_unstale":"2024-09-14T03:40:41.177732+0000","last_undegraded":"2024-09-14T03:40:41.177732+0000","last_fullsized":"2024-09-14T03:40:41.177732+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_clean_scrub_stamp":"2024-09-14T03:39:00.791161+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:24:43.135910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046880499999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":50,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-14T03:40:41.119371+0000","last_change":"2024-09-14T03:40:35.105432+0000","last_active":"2024-09-14T03:40:41.119371+0000","last_peered":"2024-09-14T03:40:41.119371+0000","last_clean":"2024-09-14T03:40:41.119371+0000","last_became_active":"2024-09-14T03:40:35.105125+0000","last_became_peered":"2024-09-14T03:40:35.105125+0000","last_unstale":"2024-09-14T03:40:41.119371+0000","last_undegraded":"2024-09-14T03:40:41.119371+0000","last_fullsized":"2024-09-14T03:40:41.119371+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_clean_scrub_stamp":"2024-09-14T03:39:00.791161+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:06:08.387860+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000765802,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"16'1","reported_seq":48,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-14T03:40:41.119494+0000","last_change":"2024-09-14T03:40:36.107793+0000","last_active":"2024-09-14T03:40:41.119494+0000","last_peered":"2024-09-14T03:40:41.119494+0000","last_clean":"2024-09-14T03:40:41.119494+0000","last_became_active":"2024-09-14T03:40:36.107394+0000","last_became_peered":"2024-09-14T03:40:36.107394+0000","last_unstale":"2024-09-14T03:40:41.119494+0000","last_undegraded":"2024-09-14T03:40:41.119494+0000","last_fullsized":"2024-09-14T03:40:41.119494+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_clean_scrub_stamp":"2024-09-14T03:39:00.791161+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:34:57.278411+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00075472199999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"21'104","reported_seq":199,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-14T03:40:41.119989+0000","last_change":"2024-09-14T03:40:33.097410+0000","last_active":"2024-09-14T03:40:41.119989+0000","last_peered":"2024-09-14T03:40:41.119989+0000","last_clean":"2024-09-14T03:40:41.119989+0000","last_became_active":"2024-09-14T03:40:33.097097+0000","last_became_peered":"2024-09-14T03:40:33.097097+0000","last_unstale":"2024-09-14T03:40:41.119989+0000","last_undegraded":"2024-09-14T03:40:41.119989+0000","last_fullsized":"2024-09-14T03:40:41.119989+0000","mapping_epoch":27,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":28,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:38:57.773685+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:57.773685+0000","last_clean_scrub_stamp":"2024-09-14T03:38:57.773685+0000","objects_scrubbed":0,"log_size":104,"log_dups_size":0,"ondisk_log_size":104,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:08:55.427294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":166,"num_read_kb":267,"num_write":149,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":4},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":166,"num_read_kb":267,"num_write":149,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":104,"ondisk_log_size":104,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":3,"up_from":29,"seq":124554051588,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27820,"kb_used_data":276,"kb_used_omap":4,"kb_used_meta":27515,"kb_avail":94344020,"statfs":{"total":96636764160,"available":96608276480,"internally_reserved":0,"allocated":282624,"data_stored":95913,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4433,"internal_metadata":28176047},"hb_peers":[0,1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":30,"seq":128849018884,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27812,"kb_used_data":272,"kb_used_omap":4,"kb_used_meta":27515,"kb_avail":94344028,"statfs":{"total":96636764160,"available":96608284672,"internally_reserved":0,"allocated":278528,"data_stored":95894,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4432,"internal_metadata":28176048},"hb_peers":[0,1,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":27,"seq":115964116996,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":28336,"kb_used_data":856,"kb_used_omap":4,"kb_used_meta":27451,"kb_avail":94343504,"statfs":{"total":96636764160,"available":96607748096,"internally_reserved":0,"allocated":876544,"data_stored":686281,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4431,"internal_metadata":28110513},"hb_peers":[0,2,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":27,"seq":115964116996,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":28396,"kb_used_data":852,"kb_used_omap":4,"kb_used_meta":27515,"kb_avail":94343444,"statfs":{"total":96636764160,"available":96607686656,"internally_reserved":0,"allocated":872448,"data_stored":686262,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4435,"internal_metadata":28176045},"hb_peers":[1,2,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T03:40:47.013 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-09-14T03:40:47.112 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:47.110+0000 7fbbf94ef640 1 -- 172.21.15.83:0/3673302469 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fbbf4132460 msgr2=0x7fbbf4134850 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:47.113 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:47.110+0000 7fbbf94ef640 1 --2- 172.21.15.83:0/3673302469 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fbbf4132460 0x7fbbf4134850 secure :-1 s=READY pgs=121 cs=0 l=1 rev1=1 crypto rx=0x7fbbec005fd0 tx=0x7fbbec02a780 comp rx=0 tx=0).stop 2024-09-14T03:40:47.113 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:47.111+0000 7fbbf94ef640 1 -- 172.21.15.83:0/3673302469 shutdown_connections 2024-09-14T03:40:47.113 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:47.111+0000 7fbbf94ef640 1 --2- 172.21.15.83:0/3673302469 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fbbf4132460 0x7fbbf4134850 unknown :-1 s=CLOSED pgs=121 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:47.113 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:47.111+0000 7fbbf94ef640 1 --2- 172.21.15.83:0/3673302469 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fbbf412fb30 0x7fbbf4131f20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:47.113 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:47.111+0000 7fbbf94ef640 1 -- 172.21.15.83:0/3673302469 >> 172.21.15.83:0/3673302469 conn(0x7fbbf409d480 msgr2=0x7fbbf409d890 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:47.113 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:47.111+0000 7fbbf94ef640 1 -- 172.21.15.83:0/3673302469 shutdown_connections 2024-09-14T03:40:47.113 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:47.111+0000 7fbbf94ef640 1 -- 172.21.15.83:0/3673302469 wait complete. 2024-09-14T03:40:47.115 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:47.111+0000 7fbbf94ef640 1 Processor -- start 2024-09-14T03:40:47.115 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:47.112+0000 7fbbf94ef640 1 -- start start 2024-09-14T03:40:47.115 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:47.112+0000 7fbbf94ef640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fbbf41294b0 0x7fbbf406f3e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:47.115 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:47.112+0000 7fbbf94ef640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fbbf412fb30 0x7fbbf406f920 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:47.115 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:47.112+0000 7fbbf94ef640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fbbf4132460 0x7fbbf406fe60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:47.116 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:47.112+0000 7fbbf94ef640 1 -- --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7fbbf4135790 con 0x7fbbf41294b0 2024-09-14T03:40:47.116 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:47.112+0000 7fbbf94ef640 1 -- --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7fbbf4135610 con 0x7fbbf4132460 2024-09-14T03:40:47.116 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:47.112+0000 7fbbf94ef640 1 -- --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7fbbf4135910 con 0x7fbbf412fb30 2024-09-14T03:40:47.116 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:47.112+0000 7fbbf2ffd640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fbbf41294b0 0x7fbbf406f3e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:47.116 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:47.112+0000 7fbbf27fc640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fbbf412fb30 0x7fbbf406f920 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:47.116 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:47.112+0000 7fbbf37fe640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fbbf4132460 0x7fbbf406fe60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:47.116 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:47.112+0000 7fbbf2ffd640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fbbf41294b0 0x7fbbf406f3e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3300/0 says I am v2:172.21.15.83:40176/0 (socket says 172.21.15.83:40176) 2024-09-14T03:40:47.116 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:47.112+0000 7fbbf2ffd640 1 -- 172.21.15.83:0/1114266779 learned_addr learned my addr 172.21.15.83:0/1114266779 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:40:47.116 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:47.112+0000 7fbbf27fc640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fbbf412fb30 0x7fbbf406f920 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3302/0 says I am v2:172.21.15.83:60672/0 (socket says 172.21.15.83:60672) 2024-09-14T03:40:47.116 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:47.112+0000 7fbbf27fc640 1 -- 172.21.15.83:0/1114266779 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fbbf4132460 msgr2=0x7fbbf406fe60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:47.116 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:47.112+0000 7fbbf27fc640 1 --2- 172.21.15.83:0/1114266779 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fbbf4132460 0x7fbbf406fe60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:47.116 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:47.112+0000 7fbbf27fc640 1 -- 172.21.15.83:0/1114266779 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fbbf41294b0 msgr2=0x7fbbf406f3e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:47.116 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:47.112+0000 7fbbf27fc640 1 --2- 172.21.15.83:0/1114266779 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fbbf41294b0 0x7fbbf406f3e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:47.116 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:47.112+0000 7fbbf27fc640 1 -- 172.21.15.83:0/1114266779 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fbbf412d170 con 0x7fbbf412fb30 2024-09-14T03:40:47.117 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:47.112+0000 7fbbf2ffd640 1 --2- 172.21.15.83:0/1114266779 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fbbf41294b0 0x7fbbf406f3e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:40:47.117 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:47.112+0000 7fbbf37fe640 1 --2- 172.21.15.83:0/1114266779 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fbbf4132460 0x7fbbf406fe60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:40:47.117 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:47.112+0000 7fbbf27fc640 1 --2- 172.21.15.83:0/1114266779 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fbbf412fb30 0x7fbbf406f920 secure :-1 s=READY pgs=64 cs=0 l=1 rev1=1 crypto rx=0x7fbbdc00e9f0 tx=0x7fbbdc00eeb0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:40:47.117 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:47.113+0000 7fbbeb7fe640 1 -- 172.21.15.83:0/1114266779 <== mon.2 v2:172.21.15.83:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fbbdc00cd80 con 0x7fbbf412fb30 2024-09-14T03:40:47.117 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:47.113+0000 7fbbeb7fe640 1 -- 172.21.15.83:0/1114266779 <== mon.2 v2:172.21.15.83:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fbbdc004020 con 0x7fbbf412fb30 2024-09-14T03:40:47.117 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:47.113+0000 7fbbf94ef640 1 -- 172.21.15.83:0/1114266779 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fbbf412c420 con 0x7fbbf412fb30 2024-09-14T03:40:47.117 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:47.113+0000 7fbbeb7fe640 1 -- 172.21.15.83:0/1114266779 <== mon.2 v2:172.21.15.83:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fbbdc004450 con 0x7fbbf412fb30 2024-09-14T03:40:47.117 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:47.113+0000 7fbbf94ef640 1 -- 172.21.15.83:0/1114266779 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7fbbf4070c40 con 0x7fbbf412fb30 2024-09-14T03:40:47.117 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:47.114+0000 7fbbf94ef640 1 -- 172.21.15.83:0/1114266779 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fbbf4129890 con 0x7fbbf412fb30 2024-09-14T03:40:47.117 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:47.114+0000 7fbbeb7fe640 1 -- 172.21.15.83:0/1114266779 <== mon.2 v2:172.21.15.83:3302/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7fbbdc0035f0 con 0x7fbbf412fb30 2024-09-14T03:40:47.118 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:47.114+0000 7fbbeb7fe640 1 -- 172.21.15.83:0/1114266779 <== mon.2 v2:172.21.15.83:3302/0 5 ==== osd_map(33..33 src has 1..33) ==== 3027+0+0 (secure 0 0 0) 0x7fbbdc0698d0 con 0x7fbbf412fb30 2024-09-14T03:40:47.119 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:47.118+0000 7fbbeb7fe640 1 -- 172.21.15.83:0/1114266779 <== mon.2 v2:172.21.15.83:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+157506 (secure 0 0 0) 0x7fbbdc068980 con 0x7fbbf412fb30 2024-09-14T03:40:47.267 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:47.266+0000 7fbbf94ef640 1 -- 172.21.15.83:0/1114266779 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) -- 0x7fbbf40008d0 con 0x7fbbf412fb30 2024-09-14T03:40:47.268 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:47.266+0000 7fbbeb7fe640 1 -- 172.21.15.83:0/1114266779 <== mon.2 v2:172.21.15.83:3302/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v33) ==== 74+0+9162 (secure 0 0 0) 0x7fbbdc009370 con 0x7fbbf412fb30 2024-09-14T03:40:47.268 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:40:47.268 INFO:teuthology.orchestra.run.smithi083.stdout:{"epoch":33,"fsid":"765298ef-df4c-4f28-af02-b45f60264ff9","created":"2024-09-14T03:38:43.954952+0000","modified":"2024-09-14T03:40:41.111243+0000","last_up_change":"2024-09-14T03:40:35.100253+0000","last_in_change":"2024-09-14T03:38:46.006167+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":8,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":4,"require_min_compat_client":"squid","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-14T03:38:56.922548+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"32","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":4,"score_stable":4,"optimal_score":0.5,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-14T03:39:00.433011+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"33","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":18,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.5,"score_stable":2.5,"optimal_score":1,"raw_score_acting":2.5,"raw_score_stable":2.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ff477614-343d-4d97-9df5-94933f9b904d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":9,"last_clean_end":21,"up_from":27,"up_thru":27,"down_at":22,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6801","nonce":3543625573}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6802","nonce":3543625573}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6804","nonce":3543625573}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6803","nonce":3543625573}]},"public_addr":"172.21.15.83:6801/3543625573","cluster_addr":"172.21.15.83:6802/3543625573","heartbeat_back_addr":"172.21.15.83:6804/3543625573","heartbeat_front_addr":"172.21.15.83:6803/3543625573","state":["exists","up"]},{"osd":1,"uuid":"562ec9a0-3f91-4a29-896d-94340e9fa36f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":11,"last_clean_end":21,"up_from":27,"up_thru":27,"down_at":23,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6805","nonce":543231167}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6806","nonce":543231167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6808","nonce":543231167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6807","nonce":543231167}]},"public_addr":"172.21.15.83:6805/543231167","cluster_addr":"172.21.15.83:6806/543231167","heartbeat_back_addr":"172.21.15.83:6808/543231167","heartbeat_front_addr":"172.21.15.83:6807/543231167","state":["exists","up"]},{"osd":2,"uuid":"8c242dda-3308-4804-b2be-9a1737c42cf6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":11,"last_clean_end":23,"up_from":30,"up_thru":30,"down_at":24,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6809","nonce":905248239}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6810","nonce":905248239}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6812","nonce":905248239}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6811","nonce":905248239}]},"public_addr":"172.21.15.83:6809/905248239","cluster_addr":"172.21.15.83:6810/905248239","heartbeat_back_addr":"172.21.15.83:6812/905248239","heartbeat_front_addr":"172.21.15.83:6811/905248239","state":["exists","up"]},{"osd":3,"uuid":"c4dd2c36-3c86-4b56-86fb-fede641a436c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":11,"last_clean_end":24,"up_from":29,"up_thru":30,"down_at":25,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6813","nonce":3860913217}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6814","nonce":3860913217}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6816","nonce":3860913217}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.83:6815","nonce":3860913217}]},"public_addr":"172.21.15.83:6813/3860913217","cluster_addr":"172.21.15.83:6814/3860913217","heartbeat_back_addr":"172.21.15.83:6816/3860913217","heartbeat_front_addr":"172.21.15.83:6815/3860913217","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:38:53.047098+0000","dead_epoch":22},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:38:53.354351+0000","dead_epoch":23},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:38:53.422284+0000","dead_epoch":24},{"osd":3,"down_stamp":"2024-09-14T03:40:28.062867+0000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:38:53.620657+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.83:0/293392365":"2024-09-15T03:40:20.028874+0000","172.21.15.83:0/4246762177":"2024-09-15T03:40:20.028874+0000","172.21.15.83:0/3623382676":"2024-09-15T03:40:20.028874+0000","172.21.15.83:6800/1934165865":"2024-09-15T03:40:20.028874+0000","172.21.15.83:0/4208089382":"2024-09-15T03:40:20.028874+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-14T03:40:47.270 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:47.268+0000 7fbbf94ef640 1 -- 172.21.15.83:0/1114266779 >> v1:172.21.15.83:6800/1803897559 conn(0x7fbbd40514b0 legacy=0x7fbbd4053970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:47.270 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:47.268+0000 7fbbf94ef640 1 -- 172.21.15.83:0/1114266779 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fbbf412fb30 msgr2=0x7fbbf406f920 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:47.270 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:47.268+0000 7fbbf94ef640 1 --2- 172.21.15.83:0/1114266779 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fbbf412fb30 0x7fbbf406f920 secure :-1 s=READY pgs=64 cs=0 l=1 rev1=1 crypto rx=0x7fbbdc00e9f0 tx=0x7fbbdc00eeb0 comp rx=0 tx=0).stop 2024-09-14T03:40:47.270 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:47.268+0000 7fbbf94ef640 1 -- 172.21.15.83:0/1114266779 shutdown_connections 2024-09-14T03:40:47.270 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:47.268+0000 7fbbf94ef640 1 --2- 172.21.15.83:0/1114266779 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7fbbf4132460 0x7fbbf406fe60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:47.270 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:47.268+0000 7fbbf94ef640 1 --2- 172.21.15.83:0/1114266779 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7fbbf412fb30 0x7fbbf406f920 unknown :-1 s=CLOSED pgs=64 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:47.270 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:47.268+0000 7fbbf94ef640 1 --2- 172.21.15.83:0/1114266779 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7fbbf41294b0 0x7fbbf406f3e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:47.270 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:47.268+0000 7fbbf94ef640 1 -- 172.21.15.83:0/1114266779 >> 172.21.15.83:0/1114266779 conn(0x7fbbf409d480 msgr2=0x7fbbf4127fe0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:47.271 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:47.268+0000 7fbbf94ef640 1 -- 172.21.15.83:0/1114266779 shutdown_connections 2024-09-14T03:40:47.271 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:47.268+0000 7fbbf94ef640 1 -- 172.21.15.83:0/1114266779 wait complete. 2024-09-14T03:40:48.282 INFO:tasks.ceph:Scrubbing osd.0 2024-09-14T03:40:48.282 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.0 config set osd_debug_deep_scrub_sleep 0 2024-09-14T03:40:48.398 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.396+0000 7f9d07f4d640 1 -- 172.21.15.83:0/3789706740 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f9d0012fb90 msgr2=0x7f9d00131f80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:48.398 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.396+0000 7f9d07f4d640 1 --2- 172.21.15.83:0/3789706740 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f9d0012fb90 0x7f9d00131f80 secure :-1 s=READY pgs=65 cs=0 l=1 rev1=1 crypto rx=0x7f9cf0009830 tx=0x7f9cf002a3f0 comp rx=0 tx=0).stop 2024-09-14T03:40:48.398 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.396+0000 7f9d07f4d640 1 -- 172.21.15.83:0/3789706740 shutdown_connections 2024-09-14T03:40:48.398 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.396+0000 7f9d07f4d640 1 --2- 172.21.15.83:0/3789706740 >> v2:172.21.15.83:3300/0 conn(0x7f9d001324c0 0x7f9d001348b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:48.398 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.396+0000 7f9d07f4d640 1 --2- 172.21.15.83:0/3789706740 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f9d0012fb90 0x7f9d00131f80 unknown :-1 s=CLOSED pgs=65 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:48.399 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.396+0000 7f9d07f4d640 1 -- 172.21.15.83:0/3789706740 >> 172.21.15.83:0/3789706740 conn(0x7f9d0009a3f0 msgr2=0x7f9d00097bd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:48.399 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.396+0000 7f9d07f4d640 1 -- 172.21.15.83:0/3789706740 shutdown_connections 2024-09-14T03:40:48.399 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.396+0000 7f9d07f4d640 1 -- 172.21.15.83:0/3789706740 wait complete. 2024-09-14T03:40:48.401 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.397+0000 7f9d07f4d640 1 Processor -- start 2024-09-14T03:40:48.401 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.397+0000 7f9d07f4d640 1 -- start start 2024-09-14T03:40:48.401 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.398+0000 7f9d07f4d640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f9d0012cd10 0x7f9d000913b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:48.401 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.398+0000 7f9d07f4d640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f9d0012fb90 0x7f9d000918f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:48.401 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.398+0000 7f9d07f4d640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f9d001324c0 0x7f9d00091e30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:48.401 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.398+0000 7f9d07f4d640 1 -- --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7f9d00135820 con 0x7f9d0012fb90 2024-09-14T03:40:48.401 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.398+0000 7f9d07f4d640 1 -- --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7f9d001356a0 con 0x7f9d001324c0 2024-09-14T03:40:48.401 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.398+0000 7f9d07f4d640 1 -- --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7f9d001359a0 con 0x7f9d0012cd10 2024-09-14T03:40:48.401 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.398+0000 7f9d054e8640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f9d0012fb90 0x7f9d000918f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:48.401 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.398+0000 7f9d05ce9640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f9d0012cd10 0x7f9d000913b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:48.401 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.398+0000 7f9d064ea640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f9d001324c0 0x7f9d00091e30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:48.402 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.398+0000 7f9d054e8640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f9d0012fb90 0x7f9d000918f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3300/0 says I am v2:172.21.15.83:40200/0 (socket says 172.21.15.83:40200) 2024-09-14T03:40:48.402 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.398+0000 7f9d054e8640 1 -- 172.21.15.83:0/3846526273 learned_addr learned my addr 172.21.15.83:0/3846526273 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:40:48.402 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.398+0000 7f9d05ce9640 1 -- 172.21.15.83:0/3846526273 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f9d001324c0 msgr2=0x7f9d00091e30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:48.402 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.398+0000 7f9d05ce9640 1 --2- 172.21.15.83:0/3846526273 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f9d001324c0 0x7f9d00091e30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:48.402 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.398+0000 7f9d05ce9640 1 -- 172.21.15.83:0/3846526273 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f9d0012fb90 msgr2=0x7f9d000918f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:48.402 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.398+0000 7f9d05ce9640 1 --2- 172.21.15.83:0/3846526273 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f9d0012fb90 0x7f9d000918f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:48.402 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.398+0000 7f9d05ce9640 1 -- 172.21.15.83:0/3846526273 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f9d000931e0 con 0x7f9d0012cd10 2024-09-14T03:40:48.402 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.398+0000 7f9d064ea640 1 --2- 172.21.15.83:0/3846526273 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f9d001324c0 0x7f9d00091e30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:40:48.402 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.398+0000 7f9d05ce9640 1 --2- 172.21.15.83:0/3846526273 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f9d0012cd10 0x7f9d000913b0 secure :-1 s=READY pgs=66 cs=0 l=1 rev1=1 crypto rx=0x7f9cfc00c430 tx=0x7f9cfc00c8f0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:40:48.402 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.398+0000 7f9cf67fc640 1 -- 172.21.15.83:0/3846526273 <== mon.2 v2:172.21.15.83:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9cfc017070 con 0x7f9d0012cd10 2024-09-14T03:40:48.402 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.398+0000 7f9cf67fc640 1 -- 172.21.15.83:0/3846526273 <== mon.2 v2:172.21.15.83:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f9cfc0053c0 con 0x7f9d0012cd10 2024-09-14T03:40:48.402 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.398+0000 7f9d07f4d640 1 -- 172.21.15.83:0/3846526273 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f9d0006f8c0 con 0x7f9d0012cd10 2024-09-14T03:40:48.402 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.398+0000 7f9cf67fc640 1 -- 172.21.15.83:0/3846526273 <== mon.2 v2:172.21.15.83:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9cfc003850 con 0x7f9d0012cd10 2024-09-14T03:40:48.402 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.398+0000 7f9d07f4d640 1 -- 172.21.15.83:0/3846526273 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7f9d001cacc0 con 0x7f9d0012cd10 2024-09-14T03:40:48.403 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.399+0000 7f9d07f4d640 1 -- 172.21.15.83:0/3846526273 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_get_version(what=osdmap handle=1) -- 0x7f9d0012d0f0 con 0x7f9d0012cd10 2024-09-14T03:40:48.403 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.400+0000 7f9cf67fc640 1 -- 172.21.15.83:0/3846526273 <== mon.2 v2:172.21.15.83:3302/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7f9cfc01a070 con 0x7f9d0012cd10 2024-09-14T03:40:48.403 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.400+0000 7f9cf67fc640 1 -- 172.21.15.83:0/3846526273 <== mon.2 v2:172.21.15.83:3302/0 5 ==== osd_map(33..33 src has 1..33) ==== 3027+0+0 (secure 0 0 0) 0x7f9cfc068370 con 0x7f9d0012cd10 2024-09-14T03:40:48.403 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.400+0000 7f9cf67fc640 1 -- 172.21.15.83:0/3846526273 --> v1:172.21.15.83:6801/3543625573 -- command(tid 1: {"prefix": "get_command_descriptions"}) -- 0x7f9ce405b9e0 con 0x7f9ce4057d30 2024-09-14T03:40:48.403 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.400+0000 7f9cf67fc640 1 -- 172.21.15.83:0/3846526273 <== mon.2 v2:172.21.15.83:3302/0 6 ==== mon_get_version_reply(handle=1 version=33) ==== 24+0+0 (secure 0 0 0) 0x7f9cfc06f050 con 0x7f9d0012cd10 2024-09-14T03:40:48.403 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.401+0000 7f9cf67fc640 1 -- 172.21.15.83:0/3846526273 <== osd.0 v1:172.21.15.83:6801/3543625573 1 ==== command_reply(tid 1: 0 ) ==== 8+0+27193 (unknown 0 0 1302949928) 0x7f9ce405b9e0 con 0x7f9ce4057d30 2024-09-14T03:40:48.415 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.414+0000 7f9d07f4d640 1 -- 172.21.15.83:0/3846526273 --> v1:172.21.15.83:6801/3543625573 -- command(tid 2: {"prefix": "config set", "var": "osd_debug_deep_scrub_sleep", "val": ["0"]}) -- 0x7f9d0012ebb0 con 0x7f9ce4057d30 2024-09-14T03:40:48.415 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.414+0000 7f9cf67fc640 1 -- 172.21.15.83:0/3846526273 <== osd.0 v1:172.21.15.83:6801/3543625573 2 ==== command_reply(tid 2: 0 ) ==== 8+0+565 (unknown 0 0 125856537) 0x7f9d0012ebb0 con 0x7f9ce4057d30 2024-09-14T03:40:48.416 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-09-14T03:40:48.416 INFO:teuthology.orchestra.run.smithi083.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) osd_delete_sleep = '' osd_delete_sleep_hdd = '' osd_delete_sleep_hybrid = '' osd_delete_sleep_ssd = '' osd_max_backfills = '' osd_recovery_max_active = '' osd_recovery_max_active_hdd = '' osd_recovery_max_active_ssd = '' osd_recovery_sleep = '' osd_recovery_sleep_hdd = '' osd_recovery_sleep_hybrid = '' osd_recovery_sleep_ssd = '' osd_scrub_sleep = '' osd_snap_trim_sleep = '' osd_snap_trim_sleep_hdd = '' osd_snap_trim_sleep_hybrid = '' osd_snap_trim_sleep_ssd = '' " 2024-09-14T03:40:48.416 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-09-14T03:40:48.417 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.414+0000 7f9d07f4d640 1 -- 172.21.15.83:0/3846526273 >> v1:172.21.15.83:6801/3543625573 conn(0x7f9ce4057d30 legacy=0x7f9ce405a170 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:48.417 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.414+0000 7f9d07f4d640 1 -- 172.21.15.83:0/3846526273 >> v1:172.21.15.83:6800/1803897559 conn(0x7f9ce40514b0 legacy=0x7f9ce4053970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:48.417 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.414+0000 7f9d07f4d640 1 -- 172.21.15.83:0/3846526273 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f9d0012cd10 msgr2=0x7f9d000913b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:48.417 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.414+0000 7f9d07f4d640 1 --2- 172.21.15.83:0/3846526273 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f9d0012cd10 0x7f9d000913b0 secure :-1 s=READY pgs=66 cs=0 l=1 rev1=1 crypto rx=0x7f9cfc00c430 tx=0x7f9cfc00c8f0 comp rx=0 tx=0).stop 2024-09-14T03:40:48.417 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.414+0000 7f9d064ea640 1 -- 172.21.15.83:0/3846526273 reap_dead start 2024-09-14T03:40:48.417 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.414+0000 7f9d07f4d640 1 -- 172.21.15.83:0/3846526273 shutdown_connections 2024-09-14T03:40:48.417 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.414+0000 7f9d07f4d640 1 -- 172.21.15.83:0/3846526273 >> 172.21.15.83:0/3846526273 conn(0x7f9d0009a3f0 msgr2=0x7f9d0009c110 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:48.417 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.414+0000 7f9d07f4d640 1 -- 172.21.15.83:0/3846526273 shutdown_connections 2024-09-14T03:40:48.417 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.414+0000 7f9d07f4d640 1 -- 172.21.15.83:0/3846526273 wait complete. 2024-09-14T03:40:48.426 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd deep-scrub 0 2024-09-14T03:40:48.527 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.525+0000 7f767d5a2640 1 -- 172.21.15.83:0/164702049 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f767812cb90 msgr2=0x7f767812cf70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:48.527 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.525+0000 7f767d5a2640 1 --2- 172.21.15.83:0/164702049 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f767812cb90 0x7f767812cf70 secure :-1 s=READY pgs=67 cs=0 l=1 rev1=1 crypto rx=0x7f7668009860 tx=0x7f766802a3d0 comp rx=0 tx=0).stop 2024-09-14T03:40:48.527 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.525+0000 7f767d5a2640 1 -- 172.21.15.83:0/164702049 shutdown_connections 2024-09-14T03:40:48.527 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.525+0000 7f767d5a2640 1 --2- 172.21.15.83:0/164702049 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f767809e280 0x7f7678129130 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:48.527 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.525+0000 7f767d5a2640 1 --2- 172.21.15.83:0/164702049 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f767812cb90 0x7f767812cf70 unknown :-1 s=CLOSED pgs=67 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:48.527 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.525+0000 7f767d5a2640 1 -- 172.21.15.83:0/164702049 >> 172.21.15.83:0/164702049 conn(0x7f7678071a10 msgr2=0x7f7678073e30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:48.527 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.525+0000 7f767d5a2640 1 -- 172.21.15.83:0/164702049 shutdown_connections 2024-09-14T03:40:48.528 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.525+0000 7f767d5a2640 1 -- 172.21.15.83:0/164702049 wait complete. 2024-09-14T03:40:48.530 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.526+0000 7f767d5a2640 1 Processor -- start 2024-09-14T03:40:48.530 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.526+0000 7f767d5a2640 1 -- start start 2024-09-14T03:40:48.530 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.527+0000 7f767d5a2640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f767809e280 0x7f7678130d10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:48.530 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.527+0000 7f767d5a2640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f7678129670 0x7f7678131250 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:48.530 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.527+0000 7f767d5a2640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f767812cb90 0x7f7678131790 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:48.530 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.527+0000 7f767d5a2640 1 -- --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7f7678135690 con 0x7f7678129670 2024-09-14T03:40:48.530 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.527+0000 7f767d5a2640 1 -- --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7f7678135510 con 0x7f767812cb90 2024-09-14T03:40:48.531 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.527+0000 7f767d5a2640 1 -- --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7f7678135810 con 0x7f767809e280 2024-09-14T03:40:48.531 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.527+0000 7f7677fff640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f767809e280 0x7f7678130d10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:48.531 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.527+0000 7f76777fe640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f7678129670 0x7f7678131250 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:48.531 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.527+0000 7f7677fff640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f767809e280 0x7f7678130d10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3302/0 says I am v2:172.21.15.83:60702/0 (socket says 172.21.15.83:60702) 2024-09-14T03:40:48.531 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.527+0000 7f767cda1640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f767812cb90 0x7f7678131790 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:48.531 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.527+0000 7f7677fff640 1 -- 172.21.15.83:0/48963787 learned_addr learned my addr 172.21.15.83:0/48963787 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:40:48.531 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.527+0000 7f76777fe640 1 -- 172.21.15.83:0/48963787 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f767809e280 msgr2=0x7f7678130d10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:48.531 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.527+0000 7f76777fe640 1 --2- 172.21.15.83:0/48963787 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f767809e280 0x7f7678130d10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:48.531 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.527+0000 7f76777fe640 1 -- 172.21.15.83:0/48963787 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f767812cb90 msgr2=0x7f7678131790 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:48.531 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.527+0000 7f76777fe640 1 --2- 172.21.15.83:0/48963787 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f767812cb90 0x7f7678131790 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:48.531 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.527+0000 7f76777fe640 1 -- 172.21.15.83:0/48963787 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f7678132430 con 0x7f7678129670 2024-09-14T03:40:48.531 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.527+0000 7f76777fe640 1 --2- 172.21.15.83:0/48963787 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f7678129670 0x7f7678131250 secure :-1 s=READY pgs=124 cs=0 l=1 rev1=1 crypto rx=0x7f766c00e9c0 tx=0x7f766c00ee80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:40:48.531 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.527+0000 7f7674ff9640 1 -- 172.21.15.83:0/48963787 <== mon.0 v2:172.21.15.83:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f766c00cd80 con 0x7f7678129670 2024-09-14T03:40:48.531 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.527+0000 7f7674ff9640 1 -- 172.21.15.83:0/48963787 <== mon.0 v2:172.21.15.83:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f766c010430 con 0x7f7678129670 2024-09-14T03:40:48.532 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.527+0000 7f767d5a2640 1 -- 172.21.15.83:0/48963787 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f7678126de0 con 0x7f7678129670 2024-09-14T03:40:48.532 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.527+0000 7f7674ff9640 1 -- 172.21.15.83:0/48963787 <== mon.0 v2:172.21.15.83:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f766c010860 con 0x7f7678129670 2024-09-14T03:40:48.532 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.528+0000 7f767d5a2640 1 -- 172.21.15.83:0/48963787 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f76781336f0 con 0x7f7678129670 2024-09-14T03:40:48.532 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.529+0000 7f767d5a2640 1 -- 172.21.15.83:0/48963787 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f76780008d0 con 0x7f7678129670 2024-09-14T03:40:48.532 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.529+0000 7f7674ff9640 1 -- 172.21.15.83:0/48963787 <== mon.0 v2:172.21.15.83:3300/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7f766c01f020 con 0x7f7678129670 2024-09-14T03:40:48.532 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.529+0000 7f7674ff9640 1 -- 172.21.15.83:0/48963787 <== mon.0 v2:172.21.15.83:3300/0 5 ==== osd_map(33..33 src has 1..33) ==== 3027+0+0 (secure 0 0 0) 0x7f766c069760 con 0x7f7678129670 2024-09-14T03:40:48.534 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.532+0000 7f7674ff9640 1 -- 172.21.15.83:0/48963787 <== mon.0 v2:172.21.15.83:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+157506 (secure 0 0 0) 0x7f766c068810 con 0x7f7678129670 2024-09-14T03:40:48.683 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.681+0000 7f767d5a2640 1 -- 172.21.15.83:0/48963787 --> v1:172.21.15.83:6800/1803897559 -- mgr_command(tid 0: {"prefix": "osd deep-scrub", "who": "0", "target": ["mon-mgr", ""]}) -- 0x7f76780697f0 con 0x7f76640514e0 2024-09-14T03:40:48.683 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.681+0000 7f7674ff9640 1 -- 172.21.15.83:0/48963787 <== mgr.14106 v1:172.21.15.83:6800/1803897559 1 ==== mgr_command_reply(tid 0: 0 instructed osd(s) 0 to deep-scrub) ==== 41+0+0 (unknown 2057319476 0 0) 0x7f76780697f0 con 0x7f76640514e0 2024-09-14T03:40:48.683 INFO:teuthology.orchestra.run.smithi083.stderr:instructed osd(s) 0 to deep-scrub 2024-09-14T03:40:48.684 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.683+0000 7f767d5a2640 1 -- 172.21.15.83:0/48963787 >> v1:172.21.15.83:6800/1803897559 conn(0x7f76640514e0 legacy=0x7f76640539a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:48.685 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.683+0000 7f767d5a2640 1 -- 172.21.15.83:0/48963787 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f7678129670 msgr2=0x7f7678131250 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:48.685 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.683+0000 7f767d5a2640 1 --2- 172.21.15.83:0/48963787 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f7678129670 0x7f7678131250 secure :-1 s=READY pgs=124 cs=0 l=1 rev1=1 crypto rx=0x7f766c00e9c0 tx=0x7f766c00ee80 comp rx=0 tx=0).stop 2024-09-14T03:40:48.685 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.683+0000 7f767d5a2640 1 -- 172.21.15.83:0/48963787 shutdown_connections 2024-09-14T03:40:48.685 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.683+0000 7f767d5a2640 1 --2- 172.21.15.83:0/48963787 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f767812cb90 0x7f7678131790 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:48.685 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.683+0000 7f767d5a2640 1 --2- 172.21.15.83:0/48963787 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f7678129670 0x7f7678131250 unknown :-1 s=CLOSED pgs=124 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:48.685 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.683+0000 7f767d5a2640 1 --2- 172.21.15.83:0/48963787 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f767809e280 0x7f7678130d10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:48.685 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.683+0000 7f767d5a2640 1 -- 172.21.15.83:0/48963787 >> 172.21.15.83:0/48963787 conn(0x7f7678071a10 msgr2=0x7f7678126ff0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:48.685 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.683+0000 7f767d5a2640 1 -- 172.21.15.83:0/48963787 shutdown_connections 2024-09-14T03:40:48.685 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.683+0000 7f767d5a2640 1 -- 172.21.15.83:0/48963787 wait complete. 2024-09-14T03:40:48.695 INFO:tasks.ceph:Scrubbing osd.1 2024-09-14T03:40:48.695 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.1 config set osd_debug_deep_scrub_sleep 0 2024-09-14T03:40:48.801 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.799+0000 7f5ec83bb640 1 -- 172.21.15.83:0/989075718 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f5ec0073b60 msgr2=0x7f5ec0073f40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:48.801 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.799+0000 7f5ec83bb640 1 --2- 172.21.15.83:0/989075718 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f5ec0073b60 0x7f5ec0073f40 secure :-1 s=READY pgs=68 cs=0 l=1 rev1=1 crypto rx=0x7f5ebc009830 tx=0x7f5ebc02a3f0 comp rx=0 tx=0).stop 2024-09-14T03:40:48.801 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.799+0000 7f5ec83bb640 1 -- 172.21.15.83:0/989075718 shutdown_connections 2024-09-14T03:40:48.801 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.799+0000 7f5ec83bb640 1 --2- 172.21.15.83:0/989075718 >> v2:172.21.15.83:3300/0 conn(0x7f5ec0096c00 0x7f5ec0136ef0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:48.801 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.799+0000 7f5ec83bb640 1 --2- 172.21.15.83:0/989075718 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f5ec0073b60 0x7f5ec0073f40 unknown :-1 s=CLOSED pgs=68 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:48.801 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.799+0000 7f5ec83bb640 1 -- 172.21.15.83:0/989075718 >> 172.21.15.83:0/989075718 conn(0x7f5ec00a1aa0 msgr2=0x7f5ec009df70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:48.801 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.799+0000 7f5ec83bb640 1 -- 172.21.15.83:0/989075718 shutdown_connections 2024-09-14T03:40:48.802 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.799+0000 7f5ec83bb640 1 -- 172.21.15.83:0/989075718 wait complete. 2024-09-14T03:40:48.802 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.800+0000 7f5ec83bb640 1 Processor -- start 2024-09-14T03:40:48.802 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.800+0000 7f5ec83bb640 1 -- start start 2024-09-14T03:40:48.804 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.800+0000 7f5ec83bb640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f5ec0073b60 0x7f5ec0090390 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:48.804 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.800+0000 7f5ec83bb640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f5ec0096c00 0x7f5ec00908d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:48.804 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.800+0000 7f5ec83bb640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f5ec009e8f0 0x7f5ec0090e10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:48.804 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.800+0000 7f5ec83bb640 1 -- --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7f5ec0137af0 con 0x7f5ec0096c00 2024-09-14T03:40:48.804 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.800+0000 7f5ec83bb640 1 -- --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7f5ec0137970 con 0x7f5ec009e8f0 2024-09-14T03:40:48.804 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.800+0000 7f5ec83bb640 1 -- --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7f5ec0137c70 con 0x7f5ec0073b60 2024-09-14T03:40:48.804 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.800+0000 7f5ec6157640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f5ec0073b60 0x7f5ec0090390 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:48.804 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.800+0000 7f5ec6157640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f5ec0073b60 0x7f5ec0090390 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3302/0 says I am v2:172.21.15.83:60730/0 (socket says 172.21.15.83:60730) 2024-09-14T03:40:48.805 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.800+0000 7f5ec6157640 1 -- 172.21.15.83:0/1446790475 learned_addr learned my addr 172.21.15.83:0/1446790475 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:40:48.805 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.800+0000 7f5ec5956640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f5ec0096c00 0x7f5ec00908d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:48.805 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.800+0000 7f5ec6958640 1 --2- 172.21.15.83:0/1446790475 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f5ec009e8f0 0x7f5ec0090e10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:48.805 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.800+0000 7f5ec6958640 1 -- 172.21.15.83:0/1446790475 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f5ec0073b60 msgr2=0x7f5ec0090390 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:48.805 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.800+0000 7f5ec6958640 1 --2- 172.21.15.83:0/1446790475 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f5ec0073b60 0x7f5ec0090390 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:48.805 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.800+0000 7f5ec6958640 1 -- 172.21.15.83:0/1446790475 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f5ec0096c00 msgr2=0x7f5ec00908d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:48.805 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.800+0000 7f5ec6958640 1 --2- 172.21.15.83:0/1446790475 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f5ec0096c00 0x7f5ec00908d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:48.805 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.800+0000 7f5ec6958640 1 -- 172.21.15.83:0/1446790475 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f5ec009ae90 con 0x7f5ec009e8f0 2024-09-14T03:40:48.805 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.800+0000 7f5ec6157640 1 --2- 172.21.15.83:0/1446790475 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f5ec0073b60 0x7f5ec0090390 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:40:48.805 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.800+0000 7f5ec5956640 1 --2- 172.21.15.83:0/1446790475 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f5ec0096c00 0x7f5ec00908d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:40:48.805 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.801+0000 7f5ec6958640 1 --2- 172.21.15.83:0/1446790475 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f5ec009e8f0 0x7f5ec0090e10 secure :-1 s=READY pgs=65 cs=0 l=1 rev1=1 crypto rx=0x7f5eb800ec50 tx=0x7f5eb800c6a0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:40:48.805 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.801+0000 7f5eb6ffd640 1 -- 172.21.15.83:0/1446790475 <== mon.1 v2:172.21.15.83:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5eb8003b70 con 0x7f5ec009e8f0 2024-09-14T03:40:48.805 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.801+0000 7f5eb6ffd640 1 -- 172.21.15.83:0/1446790475 <== mon.1 v2:172.21.15.83:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f5eb8003d10 con 0x7f5ec009e8f0 2024-09-14T03:40:48.806 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.801+0000 7f5eb6ffd640 1 -- 172.21.15.83:0/1446790475 <== mon.1 v2:172.21.15.83:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5eb8004280 con 0x7f5ec009e8f0 2024-09-14T03:40:48.806 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.801+0000 7f5ec83bb640 1 -- 172.21.15.83:0/1446790475 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f5ec0129560 con 0x7f5ec009e8f0 2024-09-14T03:40:48.806 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.801+0000 7f5ec83bb640 1 -- 172.21.15.83:0/1446790475 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7f5ec0091f30 con 0x7f5ec009e8f0 2024-09-14T03:40:48.806 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.802+0000 7f5ec83bb640 1 -- 172.21.15.83:0/1446790475 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_get_version(what=osdmap handle=1) -- 0x7f5ec0073f40 con 0x7f5ec009e8f0 2024-09-14T03:40:48.806 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.802+0000 7f5eb6ffd640 1 -- 172.21.15.83:0/1446790475 <== mon.1 v2:172.21.15.83:3301/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7f5eb8004420 con 0x7f5ec009e8f0 2024-09-14T03:40:48.806 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.802+0000 7f5eb6ffd640 1 -- 172.21.15.83:0/1446790475 <== mon.1 v2:172.21.15.83:3301/0 5 ==== osd_map(33..33 src has 1..33) ==== 3027+0+0 (secure 0 0 0) 0x7f5eb8069cd0 con 0x7f5ec009e8f0 2024-09-14T03:40:48.806 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.802+0000 7f5eb6ffd640 1 -- 172.21.15.83:0/1446790475 --> v1:172.21.15.83:6805/543231167 -- command(tid 1: {"prefix": "get_command_descriptions"}) -- 0x7f5eb8019e50 con 0x7f5ea4057d30 2024-09-14T03:40:48.806 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.802+0000 7f5eb6ffd640 1 -- 172.21.15.83:0/1446790475 <== mon.1 v2:172.21.15.83:3301/0 6 ==== mon_get_version_reply(handle=1 version=33) ==== 24+0+0 (secure 0 0 0) 0x7f5eb806c3d0 con 0x7f5ec009e8f0 2024-09-14T03:40:48.806 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.803+0000 7f5eb6ffd640 1 -- 172.21.15.83:0/1446790475 <== osd.1 v1:172.21.15.83:6805/543231167 1 ==== command_reply(tid 1: 0 ) ==== 8+0+27193 (unknown 0 0 1302949928) 0x7f5eb8019e50 con 0x7f5ea4057d30 2024-09-14T03:40:48.817 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.815+0000 7f5ec83bb640 1 -- 172.21.15.83:0/1446790475 --> v1:172.21.15.83:6805/543231167 -- command(tid 2: {"prefix": "config set", "var": "osd_debug_deep_scrub_sleep", "val": ["0"]}) -- 0x7f5ec012f320 con 0x7f5ea4057d30 2024-09-14T03:40:48.817 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.816+0000 7f5eb6ffd640 1 -- 172.21.15.83:0/1446790475 <== osd.1 v1:172.21.15.83:6805/543231167 2 ==== command_reply(tid 2: 0 ) ==== 8+0+565 (unknown 0 0 125856537) 0x7f5ec012f320 con 0x7f5ea4057d30 2024-09-14T03:40:48.817 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-09-14T03:40:48.817 INFO:teuthology.orchestra.run.smithi083.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) osd_delete_sleep = '' osd_delete_sleep_hdd = '' osd_delete_sleep_hybrid = '' osd_delete_sleep_ssd = '' osd_max_backfills = '' osd_recovery_max_active = '' osd_recovery_max_active_hdd = '' osd_recovery_max_active_ssd = '' osd_recovery_sleep = '' osd_recovery_sleep_hdd = '' osd_recovery_sleep_hybrid = '' osd_recovery_sleep_ssd = '' osd_scrub_sleep = '' osd_snap_trim_sleep = '' osd_snap_trim_sleep_hdd = '' osd_snap_trim_sleep_hybrid = '' osd_snap_trim_sleep_ssd = '' " 2024-09-14T03:40:48.818 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-09-14T03:40:48.818 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.816+0000 7f5ec83bb640 1 -- 172.21.15.83:0/1446790475 >> v1:172.21.15.83:6805/543231167 conn(0x7f5ea4057d30 legacy=0x7f5ea405a170 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:48.819 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.816+0000 7f5ec83bb640 1 -- 172.21.15.83:0/1446790475 >> v1:172.21.15.83:6800/1803897559 conn(0x7f5ea40514b0 legacy=0x7f5ea4053970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:48.819 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.816+0000 7f5ec83bb640 1 -- 172.21.15.83:0/1446790475 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f5ec009e8f0 msgr2=0x7f5ec0090e10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:48.819 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.816+0000 7f5ec83bb640 1 --2- 172.21.15.83:0/1446790475 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f5ec009e8f0 0x7f5ec0090e10 secure :-1 s=READY pgs=65 cs=0 l=1 rev1=1 crypto rx=0x7f5eb800ec50 tx=0x7f5eb800c6a0 comp rx=0 tx=0).stop 2024-09-14T03:40:48.819 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.816+0000 7f5ec6958640 1 -- 172.21.15.83:0/1446790475 reap_dead start 2024-09-14T03:40:48.819 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.816+0000 7f5ec83bb640 1 -- 172.21.15.83:0/1446790475 shutdown_connections 2024-09-14T03:40:48.819 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.816+0000 7f5ec83bb640 1 -- 172.21.15.83:0/1446790475 >> 172.21.15.83:0/1446790475 conn(0x7f5ec00a1aa0 msgr2=0x7f5ec012e9c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:48.819 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.816+0000 7f5ec83bb640 1 -- 172.21.15.83:0/1446790475 shutdown_connections 2024-09-14T03:40:48.819 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.816+0000 7f5ec83bb640 1 -- 172.21.15.83:0/1446790475 wait complete. 2024-09-14T03:40:48.828 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd deep-scrub 1 2024-09-14T03:40:48.929 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.927+0000 7f3b84590640 1 -- 172.21.15.83:0/1354265124 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f3b7c095570 msgr2=0x7f3b7c0959e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:48.929 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.927+0000 7f3b84590640 1 --2- 172.21.15.83:0/1354265124 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f3b7c095570 0x7f3b7c0959e0 secure :-1 s=READY pgs=127 cs=0 l=1 rev1=1 crypto rx=0x7f3b6c009be0 tx=0x7f3b6c02a840 comp rx=0 tx=0).stop 2024-09-14T03:40:48.929 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.927+0000 7f3b84590640 1 -- 172.21.15.83:0/1354265124 shutdown_connections 2024-09-14T03:40:48.929 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.927+0000 7f3b84590640 1 --2- 172.21.15.83:0/1354265124 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f3b7c095570 0x7f3b7c0959e0 unknown :-1 s=CLOSED pgs=127 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:48.930 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.927+0000 7f3b84590640 1 --2- 172.21.15.83:0/1354265124 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f3b7c12b490 0x7f3b7c12b870 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:48.930 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.927+0000 7f3b84590640 1 -- 172.21.15.83:0/1354265124 >> 172.21.15.83:0/1354265124 conn(0x7f3b7c09afd0 msgr2=0x7f3b7c09b3e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:48.930 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.927+0000 7f3b84590640 1 -- 172.21.15.83:0/1354265124 shutdown_connections 2024-09-14T03:40:48.930 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.927+0000 7f3b84590640 1 -- 172.21.15.83:0/1354265124 wait complete. 2024-09-14T03:40:48.932 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.928+0000 7f3b84590640 1 Processor -- start 2024-09-14T03:40:48.932 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.928+0000 7f3b84590640 1 -- start start 2024-09-14T03:40:48.932 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.929+0000 7f3b84590640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f3b7c095570 0x7f3b7c12c440 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:48.932 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.929+0000 7f3b84590640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f3b7c12b490 0x7f3b7c12c980 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:48.932 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.929+0000 7f3b84590640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f3b7c12bdb0 0x7f3b7c12cec0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:48.932 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.929+0000 7f3b84590640 1 -- --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7f3b7c133e10 con 0x7f3b7c12bdb0 2024-09-14T03:40:48.933 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.929+0000 7f3b84590640 1 -- --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7f3b7c133c90 con 0x7f3b7c095570 2024-09-14T03:40:48.933 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.929+0000 7f3b84590640 1 -- --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7f3b7c133f90 con 0x7f3b7c12b490 2024-09-14T03:40:48.933 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.929+0000 7f3b82b2d640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f3b7c12bdb0 0x7f3b7c12cec0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:48.933 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.929+0000 7f3b81b2b640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f3b7c12b490 0x7f3b7c12c980 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:48.933 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.929+0000 7f3b8232c640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f3b7c095570 0x7f3b7c12c440 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:48.933 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.929+0000 7f3b82b2d640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f3b7c12bdb0 0x7f3b7c12cec0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3300/0 says I am v2:172.21.15.83:40234/0 (socket says 172.21.15.83:40234) 2024-09-14T03:40:48.933 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.929+0000 7f3b82b2d640 1 -- 172.21.15.83:0/3964911909 learned_addr learned my addr 172.21.15.83:0/3964911909 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:40:48.933 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.929+0000 7f3b82b2d640 1 -- 172.21.15.83:0/3964911909 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f3b7c12b490 msgr2=0x7f3b7c12c980 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:48.933 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.929+0000 7f3b82b2d640 1 --2- 172.21.15.83:0/3964911909 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f3b7c12b490 0x7f3b7c12c980 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:48.934 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.929+0000 7f3b82b2d640 1 -- 172.21.15.83:0/3964911909 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f3b7c095570 msgr2=0x7f3b7c12c440 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:48.934 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.929+0000 7f3b82b2d640 1 --2- 172.21.15.83:0/3964911909 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f3b7c095570 0x7f3b7c12c440 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:48.934 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.929+0000 7f3b82b2d640 1 -- 172.21.15.83:0/3964911909 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f3b7c0741a0 con 0x7f3b7c12bdb0 2024-09-14T03:40:48.934 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.929+0000 7f3b82b2d640 1 --2- 172.21.15.83:0/3964911909 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f3b7c12bdb0 0x7f3b7c12cec0 secure :-1 s=READY pgs=128 cs=0 l=1 rev1=1 crypto rx=0x7f3b6c006010 tx=0x7f3b6c033dd0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:40:48.934 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.929+0000 7f3b7affd640 1 -- 172.21.15.83:0/3964911909 <== mon.0 v2:172.21.15.83:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3b6c042020 con 0x7f3b7c12bdb0 2024-09-14T03:40:48.934 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.929+0000 7f3b7affd640 1 -- 172.21.15.83:0/3964911909 <== mon.0 v2:172.21.15.83:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f3b6c004040 con 0x7f3b7c12bdb0 2024-09-14T03:40:48.934 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.929+0000 7f3b7affd640 1 -- 172.21.15.83:0/3964911909 <== mon.0 v2:172.21.15.83:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3b6c037040 con 0x7f3b7c12bdb0 2024-09-14T03:40:48.934 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.929+0000 7f3b84590640 1 -- 172.21.15.83:0/3964911909 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f3b7c073b70 con 0x7f3b7c12bdb0 2024-09-14T03:40:48.934 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.929+0000 7f3b84590640 1 -- 172.21.15.83:0/3964911909 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f3b7c12e050 con 0x7f3b7c12bdb0 2024-09-14T03:40:48.934 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.930+0000 7f3b84590640 1 -- 172.21.15.83:0/3964911909 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f3b7c12e280 con 0x7f3b7c12bdb0 2024-09-14T03:40:48.934 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.930+0000 7f3b7affd640 1 -- 172.21.15.83:0/3964911909 <== mon.0 v2:172.21.15.83:3300/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7f3b6c045020 con 0x7f3b7c12bdb0 2024-09-14T03:40:48.934 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.931+0000 7f3b7affd640 1 -- 172.21.15.83:0/3964911909 <== mon.0 v2:172.21.15.83:3300/0 5 ==== osd_map(33..33 src has 1..33) ==== 3027+0+0 (secure 0 0 0) 0x7f3b6c089d30 con 0x7f3b7c12bdb0 2024-09-14T03:40:48.935 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:48.934+0000 7f3b7affd640 1 -- 172.21.15.83:0/3964911909 <== mon.0 v2:172.21.15.83:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+157506 (secure 0 0 0) 0x7f3b6c04c030 con 0x7f3b7c12bdb0 2024-09-14T03:40:49.082 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.080+0000 7f3b84590640 1 -- 172.21.15.83:0/3964911909 --> v1:172.21.15.83:6800/1803897559 -- mgr_command(tid 0: {"prefix": "osd deep-scrub", "who": "1", "target": ["mon-mgr", ""]}) -- 0x7f3b7c0959e0 con 0x7f3b680514b0 2024-09-14T03:40:49.082 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.080+0000 7f3b7affd640 1 -- 172.21.15.83:0/3964911909 <== mgr.14106 v1:172.21.15.83:6800/1803897559 1 ==== mgr_command_reply(tid 0: 0 instructed osd(s) 1 to deep-scrub) ==== 41+0+0 (unknown 489605429 0 0) 0x7f3b7c0959e0 con 0x7f3b680514b0 2024-09-14T03:40:49.082 INFO:teuthology.orchestra.run.smithi083.stderr:instructed osd(s) 1 to deep-scrub 2024-09-14T03:40:49.084 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.082+0000 7f3b84590640 1 -- 172.21.15.83:0/3964911909 >> v1:172.21.15.83:6800/1803897559 conn(0x7f3b680514b0 legacy=0x7f3b68053970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:49.084 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.082+0000 7f3b84590640 1 -- 172.21.15.83:0/3964911909 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f3b7c12bdb0 msgr2=0x7f3b7c12cec0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:49.084 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.082+0000 7f3b84590640 1 --2- 172.21.15.83:0/3964911909 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f3b7c12bdb0 0x7f3b7c12cec0 secure :-1 s=READY pgs=128 cs=0 l=1 rev1=1 crypto rx=0x7f3b6c006010 tx=0x7f3b6c033dd0 comp rx=0 tx=0).stop 2024-09-14T03:40:49.084 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.082+0000 7f3b84590640 1 -- 172.21.15.83:0/3964911909 shutdown_connections 2024-09-14T03:40:49.084 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.082+0000 7f3b84590640 1 --2- 172.21.15.83:0/3964911909 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f3b7c12bdb0 0x7f3b7c12cec0 unknown :-1 s=CLOSED pgs=128 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:49.084 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.082+0000 7f3b84590640 1 --2- 172.21.15.83:0/3964911909 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f3b7c12b490 0x7f3b7c12c980 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:49.084 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.082+0000 7f3b84590640 1 --2- 172.21.15.83:0/3964911909 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f3b7c095570 0x7f3b7c12c440 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:49.084 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.082+0000 7f3b84590640 1 -- 172.21.15.83:0/3964911909 >> 172.21.15.83:0/3964911909 conn(0x7f3b7c09afd0 msgr2=0x7f3b7c0901a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:49.084 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.082+0000 7f3b84590640 1 -- 172.21.15.83:0/3964911909 shutdown_connections 2024-09-14T03:40:49.085 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.082+0000 7f3b84590640 1 -- 172.21.15.83:0/3964911909 wait complete. 2024-09-14T03:40:49.094 INFO:tasks.ceph:Scrubbing osd.2 2024-09-14T03:40:49.094 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.2 config set osd_debug_deep_scrub_sleep 0 2024-09-14T03:40:49.200 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.198+0000 7ff78d77f640 1 -- 172.21.15.83:0/2814804335 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7ff788073b60 msgr2=0x7ff788073f40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:49.200 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.198+0000 7ff78d77f640 1 --2- 172.21.15.83:0/2814804335 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7ff788073b60 0x7ff788073f40 secure :-1 s=READY pgs=130 cs=0 l=1 rev1=1 crypto rx=0x7ff778005fd0 tx=0x7ff77802a780 comp rx=0 tx=0).stop 2024-09-14T03:40:49.201 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.198+0000 7ff78d77f640 1 -- 172.21.15.83:0/2814804335 shutdown_connections 2024-09-14T03:40:49.201 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.198+0000 7ff78d77f640 1 --2- 172.21.15.83:0/2814804335 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7ff788096c00 0x7ff788136ef0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:49.201 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.198+0000 7ff78d77f640 1 --2- 172.21.15.83:0/2814804335 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7ff788073b60 0x7ff788073f40 unknown :-1 s=CLOSED pgs=130 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:49.201 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.198+0000 7ff78d77f640 1 -- 172.21.15.83:0/2814804335 >> 172.21.15.83:0/2814804335 conn(0x7ff7880a1aa0 msgr2=0x7ff78809df70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:49.201 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.198+0000 7ff78d77f640 1 -- 172.21.15.83:0/2814804335 shutdown_connections 2024-09-14T03:40:49.201 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.198+0000 7ff78d77f640 1 -- 172.21.15.83:0/2814804335 wait complete. 2024-09-14T03:40:49.202 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.199+0000 7ff78d77f640 1 Processor -- start 2024-09-14T03:40:49.202 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.199+0000 7ff78d77f640 1 -- start start 2024-09-14T03:40:49.206 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.199+0000 7ff78d77f640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7ff788073b60 0x7ff788133e80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:49.206 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.199+0000 7ff78d77f640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7ff788096c00 0x7ff7881343c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:49.206 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.199+0000 7ff78d77f640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7ff78809e8f0 0x7ff788134900 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:49.206 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.199+0000 7ff78d77f640 1 -- --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7ff788137aa0 con 0x7ff788096c00 2024-09-14T03:40:49.206 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.199+0000 7ff78d77f640 1 -- --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7ff788137920 con 0x7ff788073b60 2024-09-14T03:40:49.206 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.199+0000 7ff78d77f640 1 -- --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7ff788137c20 con 0x7ff78809e8f0 2024-09-14T03:40:49.207 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.199+0000 7ff786ffd640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7ff788073b60 0x7ff788133e80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:49.207 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.199+0000 7ff7867fc640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7ff788096c00 0x7ff7881343c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:49.207 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.199+0000 7ff786ffd640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7ff788073b60 0x7ff788133e80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3301/0 says I am v2:172.21.15.83:53962/0 (socket says 172.21.15.83:53962) 2024-09-14T03:40:49.207 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.199+0000 7ff786ffd640 1 -- 172.21.15.83:0/4144684914 learned_addr learned my addr 172.21.15.83:0/4144684914 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:40:49.207 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.199+0000 7ff7877fe640 1 --2- 172.21.15.83:0/4144684914 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7ff78809e8f0 0x7ff788134900 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:49.207 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.199+0000 7ff786ffd640 1 -- 172.21.15.83:0/4144684914 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7ff78809e8f0 msgr2=0x7ff788134900 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:49.207 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.199+0000 7ff786ffd640 1 --2- 172.21.15.83:0/4144684914 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7ff78809e8f0 0x7ff788134900 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:49.207 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.199+0000 7ff786ffd640 1 -- 172.21.15.83:0/4144684914 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7ff788096c00 msgr2=0x7ff7881343c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:49.207 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.199+0000 7ff786ffd640 1 --2- 172.21.15.83:0/4144684914 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7ff788096c00 0x7ff7881343c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:49.207 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.200+0000 7ff786ffd640 1 -- 172.21.15.83:0/4144684914 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7ff78809b640 con 0x7ff788073b60 2024-09-14T03:40:49.207 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.200+0000 7ff7867fc640 1 --2- 172.21.15.83:0/4144684914 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7ff788096c00 0x7ff7881343c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:40:49.207 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.200+0000 7ff786ffd640 1 --2- 172.21.15.83:0/4144684914 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7ff788073b60 0x7ff788133e80 secure :-1 s=READY pgs=66 cs=0 l=1 rev1=1 crypto rx=0x7ff778009b30 tx=0x7ff778003c90 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:40:49.207 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.200+0000 7ff77f7fe640 1 -- 172.21.15.83:0/4144684914 <== mon.1 v2:172.21.15.83:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff778040070 con 0x7ff788073b60 2024-09-14T03:40:49.207 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.200+0000 7ff77f7fe640 1 -- 172.21.15.83:0/4144684914 <== mon.1 v2:172.21.15.83:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7ff778004540 con 0x7ff788073b60 2024-09-14T03:40:49.208 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.200+0000 7ff78d77f640 1 -- 172.21.15.83:0/4144684914 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7ff7881366a0 con 0x7ff788073b60 2024-09-14T03:40:49.208 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.200+0000 7ff77f7fe640 1 -- 172.21.15.83:0/4144684914 <== mon.1 v2:172.21.15.83:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff77803c420 con 0x7ff788073b60 2024-09-14T03:40:49.208 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.200+0000 7ff78d77f640 1 -- 172.21.15.83:0/4144684914 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7ff7881c4f20 con 0x7ff788073b60 2024-09-14T03:40:49.208 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.201+0000 7ff78d77f640 1 -- 172.21.15.83:0/4144684914 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_get_version(what=osdmap handle=1) -- 0x7ff788135930 con 0x7ff788073b60 2024-09-14T03:40:49.208 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.201+0000 7ff77f7fe640 1 -- 172.21.15.83:0/4144684914 <== mon.1 v2:172.21.15.83:3301/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7ff778043070 con 0x7ff788073b60 2024-09-14T03:40:49.208 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.201+0000 7ff77f7fe640 1 -- 172.21.15.83:0/4144684914 <== mon.1 v2:172.21.15.83:3301/0 5 ==== osd_map(33..33 src has 1..33) ==== 3027+0+0 (secure 0 0 0) 0x7ff77808a500 con 0x7ff788073b60 2024-09-14T03:40:49.208 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.201+0000 7ff77f7fe640 1 -- 172.21.15.83:0/4144684914 --> v1:172.21.15.83:6809/905248239 -- command(tid 1: {"prefix": "get_command_descriptions"}) -- 0x7ff778044ec0 con 0x7ff768057d30 2024-09-14T03:40:49.208 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.201+0000 7ff77f7fe640 1 -- 172.21.15.83:0/4144684914 <== mon.1 v2:172.21.15.83:3301/0 6 ==== mon_get_version_reply(handle=1 version=33) ==== 24+0+0 (secure 0 0 0) 0x7ff778091050 con 0x7ff788073b60 2024-09-14T03:40:49.208 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.202+0000 7ff77f7fe640 1 -- 172.21.15.83:0/4144684914 <== osd.2 v1:172.21.15.83:6809/905248239 1 ==== command_reply(tid 1: 0 ) ==== 8+0+27193 (unknown 0 0 1302949928) 0x7ff778044ec0 con 0x7ff768057d30 2024-09-14T03:40:49.216 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.215+0000 7ff78d77f640 1 -- 172.21.15.83:0/4144684914 --> v1:172.21.15.83:6809/905248239 -- command(tid 2: {"prefix": "config set", "var": "osd_debug_deep_scrub_sleep", "val": ["0"]}) -- 0x7ff78809cb40 con 0x7ff768057d30 2024-09-14T03:40:49.216 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.215+0000 7ff77f7fe640 1 -- 172.21.15.83:0/4144684914 <== osd.2 v1:172.21.15.83:6809/905248239 2 ==== command_reply(tid 2: 0 ) ==== 8+0+565 (unknown 0 0 125856537) 0x7ff78809cb40 con 0x7ff768057d30 2024-09-14T03:40:49.216 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-09-14T03:40:49.217 INFO:teuthology.orchestra.run.smithi083.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) osd_delete_sleep = '' osd_delete_sleep_hdd = '' osd_delete_sleep_hybrid = '' osd_delete_sleep_ssd = '' osd_max_backfills = '' osd_recovery_max_active = '' osd_recovery_max_active_hdd = '' osd_recovery_max_active_ssd = '' osd_recovery_sleep = '' osd_recovery_sleep_hdd = '' osd_recovery_sleep_hybrid = '' osd_recovery_sleep_ssd = '' osd_scrub_sleep = '' osd_snap_trim_sleep = '' osd_snap_trim_sleep_hdd = '' osd_snap_trim_sleep_hybrid = '' osd_snap_trim_sleep_ssd = '' " 2024-09-14T03:40:49.217 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-09-14T03:40:49.218 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.215+0000 7ff78d77f640 1 -- 172.21.15.83:0/4144684914 >> v1:172.21.15.83:6809/905248239 conn(0x7ff768057d30 legacy=0x7ff76805a170 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:49.218 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.215+0000 7ff78d77f640 1 -- 172.21.15.83:0/4144684914 >> v1:172.21.15.83:6800/1803897559 conn(0x7ff7680514b0 legacy=0x7ff768053970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:49.218 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.215+0000 7ff78d77f640 1 -- 172.21.15.83:0/4144684914 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7ff788073b60 msgr2=0x7ff788133e80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:49.218 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.215+0000 7ff78d77f640 1 --2- 172.21.15.83:0/4144684914 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7ff788073b60 0x7ff788133e80 secure :-1 s=READY pgs=66 cs=0 l=1 rev1=1 crypto rx=0x7ff778009b30 tx=0x7ff778003c90 comp rx=0 tx=0).stop 2024-09-14T03:40:49.218 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.215+0000 7ff7877fe640 1 -- 172.21.15.83:0/4144684914 reap_dead start 2024-09-14T03:40:49.218 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.215+0000 7ff78d77f640 1 -- 172.21.15.83:0/4144684914 shutdown_connections 2024-09-14T03:40:49.218 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.215+0000 7ff78d77f640 1 -- 172.21.15.83:0/4144684914 >> 172.21.15.83:0/4144684914 conn(0x7ff7880a1aa0 msgr2=0x7ff78809c1e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:49.218 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.215+0000 7ff78d77f640 1 -- 172.21.15.83:0/4144684914 shutdown_connections 2024-09-14T03:40:49.218 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.215+0000 7ff78d77f640 1 -- 172.21.15.83:0/4144684914 wait complete. 2024-09-14T03:40:49.227 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd deep-scrub 2 2024-09-14T03:40:49.327 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.325+0000 7f7d4b01d640 1 -- 172.21.15.83:0/1081959090 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f7d4412f1c0 msgr2=0x7f7d4412f600 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:49.327 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.325+0000 7f7d4b01d640 1 --2- 172.21.15.83:0/1081959090 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f7d4412f1c0 0x7f7d4412f600 secure :-1 s=READY pgs=132 cs=0 l=1 rev1=1 crypto rx=0x7f7d3c005fd0 tx=0x7f7d3c02a780 comp rx=0 tx=0).stop 2024-09-14T03:40:49.327 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.325+0000 7f7d4b01d640 1 -- 172.21.15.83:0/1081959090 shutdown_connections 2024-09-14T03:40:49.327 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.325+0000 7f7d4b01d640 1 --2- 172.21.15.83:0/1081959090 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f7d4412f1c0 0x7f7d4412f600 unknown :-1 s=CLOSED pgs=132 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:49.328 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.325+0000 7f7d4b01d640 1 --2- 172.21.15.83:0/1081959090 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f7d44098d40 0x7f7d44099120 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:49.328 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.325+0000 7f7d4b01d640 1 -- 172.21.15.83:0/1081959090 >> 172.21.15.83:0/1081959090 conn(0x7f7d440a1790 msgr2=0x7f7d440a1ba0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:49.328 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.326+0000 7f7d4b01d640 1 -- 172.21.15.83:0/1081959090 shutdown_connections 2024-09-14T03:40:49.328 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.326+0000 7f7d4b01d640 1 -- 172.21.15.83:0/1081959090 wait complete. 2024-09-14T03:40:49.330 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.326+0000 7f7d4b01d640 1 Processor -- start 2024-09-14T03:40:49.330 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.326+0000 7f7d4b01d640 1 -- start start 2024-09-14T03:40:49.330 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.327+0000 7f7d4b01d640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f7d44098d40 0x7f7d441c4f30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:49.330 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.327+0000 7f7d4b01d640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f7d44099660 0x7f7d441c5470 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:49.330 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.327+0000 7f7d4b01d640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f7d4412f1c0 0x7f7d440714b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:49.330 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.327+0000 7f7d4b01d640 1 -- --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7f7d44137870 con 0x7f7d4412f1c0 2024-09-14T03:40:49.330 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.327+0000 7f7d4b01d640 1 -- --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7f7d441376f0 con 0x7f7d44099660 2024-09-14T03:40:49.331 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.327+0000 7f7d4b01d640 1 -- --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7f7d441379f0 con 0x7f7d44098d40 2024-09-14T03:40:49.331 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.327+0000 7f7d48db9640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f7d44098d40 0x7f7d441c4f30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:49.331 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.327+0000 7f7d43fff640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f7d44099660 0x7f7d441c5470 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:49.331 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.327+0000 7f7d495ba640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f7d4412f1c0 0x7f7d440714b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:49.331 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.327+0000 7f7d495ba640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f7d4412f1c0 0x7f7d440714b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3300/0 says I am v2:172.21.15.83:40276/0 (socket says 172.21.15.83:40276) 2024-09-14T03:40:49.331 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.327+0000 7f7d48db9640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f7d44098d40 0x7f7d441c4f30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3302/0 says I am v2:172.21.15.83:60776/0 (socket says 172.21.15.83:60776) 2024-09-14T03:40:49.331 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.327+0000 7f7d43fff640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f7d44099660 0x7f7d441c5470 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3301/0 says I am v2:172.21.15.83:53984/0 (socket says 172.21.15.83:53984) 2024-09-14T03:40:49.331 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.327+0000 7f7d495ba640 1 -- 172.21.15.83:0/1380647095 learned_addr learned my addr 172.21.15.83:0/1380647095 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:40:49.331 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.327+0000 7f7d43fff640 1 -- 172.21.15.83:0/1380647095 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f7d44098d40 msgr2=0x7f7d441c4f30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:49.331 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.327+0000 7f7d43fff640 1 --2- 172.21.15.83:0/1380647095 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f7d44098d40 0x7f7d441c4f30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:49.331 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.327+0000 7f7d43fff640 1 -- 172.21.15.83:0/1380647095 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f7d4412f1c0 msgr2=0x7f7d440714b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:49.331 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.327+0000 7f7d43fff640 1 --2- 172.21.15.83:0/1380647095 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f7d4412f1c0 0x7f7d440714b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:49.331 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.327+0000 7f7d43fff640 1 -- 172.21.15.83:0/1380647095 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f7d44072e40 con 0x7f7d44099660 2024-09-14T03:40:49.331 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.327+0000 7f7d48db9640 1 --2- 172.21.15.83:0/1380647095 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f7d44098d40 0x7f7d441c4f30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:40:49.332 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.327+0000 7f7d495ba640 1 --2- 172.21.15.83:0/1380647095 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f7d4412f1c0 0x7f7d440714b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:40:49.332 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.327+0000 7f7d43fff640 1 --2- 172.21.15.83:0/1380647095 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f7d44099660 0x7f7d441c5470 secure :-1 s=READY pgs=67 cs=0 l=1 rev1=1 crypto rx=0x7f7d3800c430 tx=0x7f7d3800c8f0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:40:49.332 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.328+0000 7f7d417fa640 1 -- 172.21.15.83:0/1380647095 <== mon.1 v2:172.21.15.83:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7d38017070 con 0x7f7d44099660 2024-09-14T03:40:49.332 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.328+0000 7f7d417fa640 1 -- 172.21.15.83:0/1380647095 <== mon.1 v2:172.21.15.83:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f7d380053c0 con 0x7f7d44099660 2024-09-14T03:40:49.332 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.328+0000 7f7d417fa640 1 -- 172.21.15.83:0/1380647095 <== mon.1 v2:172.21.15.83:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7d38003850 con 0x7f7d44099660 2024-09-14T03:40:49.332 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.328+0000 7f7d4b01d640 1 -- 172.21.15.83:0/1380647095 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f7d44073720 con 0x7f7d44099660 2024-09-14T03:40:49.332 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.328+0000 7f7d4b01d640 1 -- 172.21.15.83:0/1380647095 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7f7d441c59b0 con 0x7f7d44099660 2024-09-14T03:40:49.332 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.329+0000 7f7d4b01d640 1 -- 172.21.15.83:0/1380647095 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f7d44099120 con 0x7f7d44099660 2024-09-14T03:40:49.332 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.329+0000 7f7d417fa640 1 -- 172.21.15.83:0/1380647095 <== mon.1 v2:172.21.15.83:3301/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7f7d3801a070 con 0x7f7d44099660 2024-09-14T03:40:49.332 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.329+0000 7f7d417fa640 1 -- 172.21.15.83:0/1380647095 <== mon.1 v2:172.21.15.83:3301/0 5 ==== osd_map(33..33 src has 1..33) ==== 3027+0+0 (secure 0 0 0) 0x7f7d380683b0 con 0x7f7d44099660 2024-09-14T03:40:49.334 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.333+0000 7f7d417fa640 1 -- 172.21.15.83:0/1380647095 <== mon.1 v2:172.21.15.83:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+157506 (secure 0 0 0) 0x7f7d38067460 con 0x7f7d44099660 2024-09-14T03:40:49.484 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.482+0000 7f7d4b01d640 1 -- 172.21.15.83:0/1380647095 --> v1:172.21.15.83:6800/1803897559 -- mgr_command(tid 0: {"prefix": "osd deep-scrub", "who": "2", "target": ["mon-mgr", ""]}) -- 0x7f7d440698c0 con 0x7f7d300514b0 2024-09-14T03:40:49.484 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.482+0000 7f7d417fa640 1 -- 172.21.15.83:0/1380647095 <== mgr.14106 v1:172.21.15.83:6800/1803897559 1 ==== mgr_command_reply(tid 0: 0 instructed osd(s) 2 to deep-scrub) ==== 41+0+0 (unknown 3049115190 0 0) 0x7f7d440698c0 con 0x7f7d300514b0 2024-09-14T03:40:49.484 INFO:teuthology.orchestra.run.smithi083.stderr:instructed osd(s) 2 to deep-scrub 2024-09-14T03:40:49.486 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.484+0000 7f7d4b01d640 1 -- 172.21.15.83:0/1380647095 >> v1:172.21.15.83:6800/1803897559 conn(0x7f7d300514b0 legacy=0x7f7d30053970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:49.486 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.484+0000 7f7d4b01d640 1 -- 172.21.15.83:0/1380647095 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f7d44099660 msgr2=0x7f7d441c5470 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:49.486 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.484+0000 7f7d4b01d640 1 --2- 172.21.15.83:0/1380647095 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f7d44099660 0x7f7d441c5470 secure :-1 s=READY pgs=67 cs=0 l=1 rev1=1 crypto rx=0x7f7d3800c430 tx=0x7f7d3800c8f0 comp rx=0 tx=0).stop 2024-09-14T03:40:49.486 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.484+0000 7f7d4b01d640 1 -- 172.21.15.83:0/1380647095 shutdown_connections 2024-09-14T03:40:49.486 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.484+0000 7f7d4b01d640 1 --2- 172.21.15.83:0/1380647095 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f7d4412f1c0 0x7f7d440714b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:49.486 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.484+0000 7f7d4b01d640 1 --2- 172.21.15.83:0/1380647095 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f7d44099660 0x7f7d441c5470 unknown :-1 s=CLOSED pgs=67 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:49.486 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.484+0000 7f7d4b01d640 1 --2- 172.21.15.83:0/1380647095 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f7d44098d40 0x7f7d441c4f30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:49.486 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.484+0000 7f7d4b01d640 1 -- 172.21.15.83:0/1380647095 >> 172.21.15.83:0/1380647095 conn(0x7f7d440a1790 msgr2=0x7f7d4409cc70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:49.486 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.484+0000 7f7d4b01d640 1 -- 172.21.15.83:0/1380647095 shutdown_connections 2024-09-14T03:40:49.486 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.484+0000 7f7d4b01d640 1 -- 172.21.15.83:0/1380647095 wait complete. 2024-09-14T03:40:49.497 INFO:tasks.ceph:Scrubbing osd.3 2024-09-14T03:40:49.497 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.3 config set osd_debug_deep_scrub_sleep 0 2024-09-14T03:40:49.601 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.599+0000 7f0561750640 1 -- 172.21.15.83:0/3330498237 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f055c070bd0 msgr2=0x7f055c097740 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:49.601 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.599+0000 7f0561750640 1 --2- 172.21.15.83:0/3330498237 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f055c070bd0 0x7f055c097740 secure :-1 s=READY pgs=134 cs=0 l=1 rev1=1 crypto rx=0x7f0554005fd0 tx=0x7f055402a780 comp rx=0 tx=0).stop 2024-09-14T03:40:49.601 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.599+0000 7f0561750640 1 -- 172.21.15.83:0/3330498237 shutdown_connections 2024-09-14T03:40:49.601 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.599+0000 7f0561750640 1 --2- 172.21.15.83:0/3330498237 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f055c070bd0 0x7f055c097740 unknown :-1 s=CLOSED pgs=134 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:49.601 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.599+0000 7f0561750640 1 --2- 172.21.15.83:0/3330498237 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f055c0702b0 0x7f055c070690 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:49.601 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.599+0000 7f0561750640 1 -- 172.21.15.83:0/3330498237 >> 172.21.15.83:0/3330498237 conn(0x7f055c09dcf0 msgr2=0x7f055c09e100 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:49.601 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.599+0000 7f0561750640 1 -- 172.21.15.83:0/3330498237 shutdown_connections 2024-09-14T03:40:49.601 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.599+0000 7f0561750640 1 -- 172.21.15.83:0/3330498237 wait complete. 2024-09-14T03:40:49.606 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.600+0000 7f0561750640 1 Processor -- start 2024-09-14T03:40:49.606 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.600+0000 7f0561750640 1 -- start start 2024-09-14T03:40:49.606 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.600+0000 7f0561750640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f055c0702b0 0x7f055c08cd60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:49.606 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.600+0000 7f0561750640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f055c070bd0 0x7f055c08d2a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:49.606 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.600+0000 7f0561750640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f055c076290 0x7f055c08d7e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:49.606 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.600+0000 7f0561750640 1 -- --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7f055c134700 con 0x7f055c070bd0 2024-09-14T03:40:49.607 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.600+0000 7f0561750640 1 -- --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7f055c134580 con 0x7f055c076290 2024-09-14T03:40:49.607 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.600+0000 7f0561750640 1 -- --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7f055c134880 con 0x7f055c0702b0 2024-09-14T03:40:49.607 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.600+0000 7f055affd640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f055c0702b0 0x7f055c08cd60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:49.607 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.600+0000 7f0553fff640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f055c070bd0 0x7f055c08d2a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:49.607 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.600+0000 7f055b7fe640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f055c076290 0x7f055c08d7e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:49.607 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.600+0000 7f0553fff640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f055c070bd0 0x7f055c08d2a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3300/0 says I am v2:172.21.15.83:40302/0 (socket says 172.21.15.83:40302) 2024-09-14T03:40:49.607 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.600+0000 7f055affd640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f055c0702b0 0x7f055c08cd60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3302/0 says I am v2:172.21.15.83:60784/0 (socket says 172.21.15.83:60784) 2024-09-14T03:40:49.607 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.600+0000 7f0553fff640 1 -- 172.21.15.83:0/3040737728 learned_addr learned my addr 172.21.15.83:0/3040737728 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:40:49.607 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.601+0000 7f055affd640 1 -- 172.21.15.83:0/3040737728 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f055c076290 msgr2=0x7f055c08d7e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:49.607 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.601+0000 7f055affd640 1 --2- 172.21.15.83:0/3040737728 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f055c076290 0x7f055c08d7e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:49.607 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.601+0000 7f055affd640 1 -- 172.21.15.83:0/3040737728 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f055c070bd0 msgr2=0x7f055c08d2a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:49.607 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.601+0000 7f055affd640 1 --2- 172.21.15.83:0/3040737728 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f055c070bd0 0x7f055c08d2a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:49.607 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.601+0000 7f055affd640 1 -- 172.21.15.83:0/3040737728 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f055c08e480 con 0x7f055c0702b0 2024-09-14T03:40:49.608 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.601+0000 7f055b7fe640 1 --2- 172.21.15.83:0/3040737728 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f055c076290 0x7f055c08d7e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:40:49.608 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.601+0000 7f0553fff640 1 --2- 172.21.15.83:0/3040737728 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f055c070bd0 0x7f055c08d2a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:40:49.608 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.601+0000 7f055affd640 1 --2- 172.21.15.83:0/3040737728 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f055c0702b0 0x7f055c08cd60 secure :-1 s=READY pgs=69 cs=0 l=1 rev1=1 crypto rx=0x7f054c00ea20 tx=0x7f054c00eee0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:40:49.608 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.601+0000 7f05537fe640 1 -- 172.21.15.83:0/3040737728 <== mon.2 v2:172.21.15.83:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f054c00cd50 con 0x7f055c0702b0 2024-09-14T03:40:49.608 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.601+0000 7f05537fe640 1 -- 172.21.15.83:0/3040737728 <== mon.2 v2:172.21.15.83:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f054c004020 con 0x7f055c0702b0 2024-09-14T03:40:49.608 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.601+0000 7f0561750640 1 -- 172.21.15.83:0/3040737728 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f055c08dd20 con 0x7f055c0702b0 2024-09-14T03:40:49.608 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.601+0000 7f05537fe640 1 -- 172.21.15.83:0/3040737728 <== mon.2 v2:172.21.15.83:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f054c004450 con 0x7f055c0702b0 2024-09-14T03:40:49.609 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.601+0000 7f0561750640 1 -- 172.21.15.83:0/3040737728 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7f055c1c8b90 con 0x7f055c0702b0 2024-09-14T03:40:49.609 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.602+0000 7f0561750640 1 -- 172.21.15.83:0/3040737728 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_get_version(what=osdmap handle=1) -- 0x7f055c073a00 con 0x7f055c0702b0 2024-09-14T03:40:49.609 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.602+0000 7f05537fe640 1 -- 172.21.15.83:0/3040737728 <== mon.2 v2:172.21.15.83:3302/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7f054c0035f0 con 0x7f055c0702b0 2024-09-14T03:40:49.609 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.603+0000 7f05537fe640 1 -- 172.21.15.83:0/3040737728 <== mon.2 v2:172.21.15.83:3302/0 5 ==== osd_map(33..33 src has 1..33) ==== 3027+0+0 (secure 0 0 0) 0x7f054c069890 con 0x7f055c0702b0 2024-09-14T03:40:49.610 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.603+0000 7f05537fe640 1 -- 172.21.15.83:0/3040737728 --> v1:172.21.15.83:6813/3860913217 -- command(tid 1: {"prefix": "get_command_descriptions"}) -- 0x7f053c05ba10 con 0x7f053c057d60 2024-09-14T03:40:49.610 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.603+0000 7f05537fe640 1 -- 172.21.15.83:0/3040737728 <== mon.2 v2:172.21.15.83:3302/0 6 ==== mon_get_version_reply(handle=1 version=33) ==== 24+0+0 (secure 0 0 0) 0x7f054c06e2c0 con 0x7f055c0702b0 2024-09-14T03:40:49.610 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.604+0000 7f05537fe640 1 -- 172.21.15.83:0/3040737728 <== osd.3 v1:172.21.15.83:6813/3860913217 1 ==== command_reply(tid 1: 0 ) ==== 8+0+27193 (unknown 0 0 1302949928) 0x7f053c05ba10 con 0x7f053c057d60 2024-09-14T03:40:49.617 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.616+0000 7f0561750640 1 -- 172.21.15.83:0/3040737728 --> v1:172.21.15.83:6813/3860913217 -- command(tid 2: {"prefix": "config set", "var": "osd_debug_deep_scrub_sleep", "val": ["0"]}) -- 0x7f055c096a90 con 0x7f053c057d60 2024-09-14T03:40:49.617 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.616+0000 7f05537fe640 1 -- 172.21.15.83:0/3040737728 <== osd.3 v1:172.21.15.83:6813/3860913217 2 ==== command_reply(tid 2: 0 ) ==== 8+0+565 (unknown 0 0 125856537) 0x7f055c096a90 con 0x7f053c057d60 2024-09-14T03:40:49.617 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-09-14T03:40:49.618 INFO:teuthology.orchestra.run.smithi083.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) osd_delete_sleep = '' osd_delete_sleep_hdd = '' osd_delete_sleep_hybrid = '' osd_delete_sleep_ssd = '' osd_max_backfills = '' osd_recovery_max_active = '' osd_recovery_max_active_hdd = '' osd_recovery_max_active_ssd = '' osd_recovery_sleep = '' osd_recovery_sleep_hdd = '' osd_recovery_sleep_hybrid = '' osd_recovery_sleep_ssd = '' osd_scrub_sleep = '' osd_snap_trim_sleep = '' osd_snap_trim_sleep_hdd = '' osd_snap_trim_sleep_hybrid = '' osd_snap_trim_sleep_ssd = '' " 2024-09-14T03:40:49.618 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-09-14T03:40:49.619 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.616+0000 7f0561750640 1 -- 172.21.15.83:0/3040737728 >> v1:172.21.15.83:6813/3860913217 conn(0x7f053c057d60 legacy=0x7f053c05a1a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:49.619 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.616+0000 7f0561750640 1 -- 172.21.15.83:0/3040737728 >> v1:172.21.15.83:6800/1803897559 conn(0x7f053c0514e0 legacy=0x7f053c0539a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:49.619 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.616+0000 7f0561750640 1 -- 172.21.15.83:0/3040737728 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f055c0702b0 msgr2=0x7f055c08cd60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:49.619 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.616+0000 7f0561750640 1 --2- 172.21.15.83:0/3040737728 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f055c0702b0 0x7f055c08cd60 secure :-1 s=READY pgs=69 cs=0 l=1 rev1=1 crypto rx=0x7f054c00ea20 tx=0x7f054c00eee0 comp rx=0 tx=0).stop 2024-09-14T03:40:49.619 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.616+0000 7f055b7fe640 1 -- 172.21.15.83:0/3040737728 reap_dead start 2024-09-14T03:40:49.619 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.616+0000 7f0561750640 1 -- 172.21.15.83:0/3040737728 shutdown_connections 2024-09-14T03:40:49.619 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.616+0000 7f0561750640 1 -- 172.21.15.83:0/3040737728 >> 172.21.15.83:0/3040737728 conn(0x7f055c09dcf0 msgr2=0x7f055c096130 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:49.620 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.616+0000 7f0561750640 1 -- 172.21.15.83:0/3040737728 shutdown_connections 2024-09-14T03:40:49.620 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.616+0000 7f0561750640 1 -- 172.21.15.83:0/3040737728 wait complete. 2024-09-14T03:40:49.629 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd deep-scrub 3 2024-09-14T03:40:49.731 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.729+0000 7f8a123e7640 1 -- 172.21.15.83:0/42592820 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f8a0c12eff0 msgr2=0x7f8a0c12f480 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:49.731 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.729+0000 7f8a123e7640 1 --2- 172.21.15.83:0/42592820 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f8a0c12eff0 0x7f8a0c12f480 secure :-1 s=READY pgs=68 cs=0 l=1 rev1=1 crypto rx=0x7f8a04005fd0 tx=0x7f8a0402a6e0 comp rx=0 tx=0).stop 2024-09-14T03:40:49.732 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.729+0000 7f8a123e7640 1 -- 172.21.15.83:0/42592820 shutdown_connections 2024-09-14T03:40:49.732 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.729+0000 7f8a123e7640 1 --2- 172.21.15.83:0/42592820 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f8a0c12eff0 0x7f8a0c12f480 unknown :-1 s=CLOSED pgs=68 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:49.732 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.729+0000 7f8a123e7640 1 --2- 172.21.15.83:0/42592820 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f8a0c099660 0x7f8a0c12eab0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:49.732 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.729+0000 7f8a123e7640 1 --2- 172.21.15.83:0/42592820 >> v2:172.21.15.83:3300/0 conn(0x7f8a0c098d40 0x7f8a0c099120 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:49.732 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.729+0000 7f8a123e7640 1 -- 172.21.15.83:0/42592820 >> 172.21.15.83:0/42592820 conn(0x7f8a0c0a1790 msgr2=0x7f8a0c0a1ba0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:49.732 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.729+0000 7f8a123e7640 1 -- 172.21.15.83:0/42592820 shutdown_connections 2024-09-14T03:40:49.732 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.729+0000 7f8a123e7640 1 -- 172.21.15.83:0/42592820 wait complete. 2024-09-14T03:40:49.732 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.730+0000 7f8a123e7640 1 Processor -- start 2024-09-14T03:40:49.734 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.730+0000 7f8a123e7640 1 -- start start 2024-09-14T03:40:49.734 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.731+0000 7f8a123e7640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f8a0c098d40 0x7f8a0c1c4a40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:49.735 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.731+0000 7f8a123e7640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f8a0c099660 0x7f8a0c1c4f80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:49.735 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.731+0000 7f8a123e7640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f8a0c12eff0 0x7f8a0c1c54c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:49.735 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.731+0000 7f8a123e7640 1 -- --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7f8a0c1365b0 con 0x7f8a0c099660 2024-09-14T03:40:49.735 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.731+0000 7f8a123e7640 1 -- --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7f8a0c136430 con 0x7f8a0c098d40 2024-09-14T03:40:49.735 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.731+0000 7f8a123e7640 1 -- --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7f8a0c136730 con 0x7f8a0c12eff0 2024-09-14T03:40:49.735 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.731+0000 7f8a0bfff640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f8a0c098d40 0x7f8a0c1c4a40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:49.735 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.731+0000 7f8a0bfff640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f8a0c098d40 0x7f8a0c1c4a40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3301/0 says I am v2:172.21.15.83:54026/0 (socket says 172.21.15.83:54026) 2024-09-14T03:40:49.735 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.731+0000 7f8a0b7fe640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f8a0c099660 0x7f8a0c1c4f80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:49.735 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.731+0000 7f8a0bfff640 1 -- 172.21.15.83:0/1934497398 learned_addr learned my addr 172.21.15.83:0/1934497398 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:40:49.735 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.731+0000 7f8a10984640 1 --2- 172.21.15.83:0/1934497398 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f8a0c12eff0 0x7f8a0c1c54c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:49.735 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.731+0000 7f8a0bfff640 1 -- 172.21.15.83:0/1934497398 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f8a0c12eff0 msgr2=0x7f8a0c1c54c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:49.735 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.731+0000 7f8a0bfff640 1 --2- 172.21.15.83:0/1934497398 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f8a0c12eff0 0x7f8a0c1c54c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:49.735 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.731+0000 7f8a0bfff640 1 -- 172.21.15.83:0/1934497398 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f8a0c099660 msgr2=0x7f8a0c1c4f80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:49.736 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.731+0000 7f8a0bfff640 1 --2- 172.21.15.83:0/1934497398 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f8a0c099660 0x7f8a0c1c4f80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:49.736 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.731+0000 7f8a0bfff640 1 -- 172.21.15.83:0/1934497398 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f8a0c129460 con 0x7f8a0c098d40 2024-09-14T03:40:49.736 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.731+0000 7f8a0b7fe640 1 --2- 172.21.15.83:0/1934497398 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f8a0c099660 0x7f8a0c1c4f80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:40:49.736 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.731+0000 7f8a10984640 1 --2- 172.21.15.83:0/1934497398 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f8a0c12eff0 0x7f8a0c1c54c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:40:49.736 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.731+0000 7f8a0bfff640 1 --2- 172.21.15.83:0/1934497398 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f8a0c098d40 0x7f8a0c1c4a40 secure :-1 s=READY pgs=69 cs=0 l=1 rev1=1 crypto rx=0x7f89fc00e9c0 tx=0x7f89fc00ee80 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:40:49.736 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.731+0000 7f8a08ff9640 1 -- 172.21.15.83:0/1934497398 <== mon.1 v2:172.21.15.83:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f89fc00cd80 con 0x7f8a0c098d40 2024-09-14T03:40:49.736 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.732+0000 7f8a08ff9640 1 -- 172.21.15.83:0/1934497398 <== mon.1 v2:172.21.15.83:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f89fc004020 con 0x7f8a0c098d40 2024-09-14T03:40:49.736 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.732+0000 7f8a08ff9640 1 -- 172.21.15.83:0/1934497398 <== mon.1 v2:172.21.15.83:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f89fc004450 con 0x7f8a0c098d40 2024-09-14T03:40:49.736 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.732+0000 7f8a123e7640 1 -- 172.21.15.83:0/1934497398 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f8a0c06ea40 con 0x7f8a0c098d40 2024-09-14T03:40:49.736 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.732+0000 7f8a123e7640 1 -- 172.21.15.83:0/1934497398 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7f8a0c132b20 con 0x7f8a0c098d40 2024-09-14T03:40:49.736 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.733+0000 7f8a08ff9640 1 -- 172.21.15.83:0/1934497398 <== mon.1 v2:172.21.15.83:3301/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7f89fc0035f0 con 0x7f8a0c098d40 2024-09-14T03:40:49.736 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.733+0000 7f8a123e7640 1 -- 172.21.15.83:0/1934497398 --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f8a0c12eab0 con 0x7f8a0c098d40 2024-09-14T03:40:49.737 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.733+0000 7f8a08ff9640 1 -- 172.21.15.83:0/1934497398 <== mon.1 v2:172.21.15.83:3301/0 5 ==== osd_map(33..33 src has 1..33) ==== 3027+0+0 (secure 0 0 0) 0x7f89fc069890 con 0x7f8a0c098d40 2024-09-14T03:40:49.738 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.736+0000 7f8a08ff9640 1 -- 172.21.15.83:0/1934497398 <== mon.1 v2:172.21.15.83:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+157506 (secure 0 0 0) 0x7f89fc068940 con 0x7f8a0c098d40 2024-09-14T03:40:49.887 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.885+0000 7f8a123e7640 1 -- 172.21.15.83:0/1934497398 --> v1:172.21.15.83:6800/1803897559 -- mgr_command(tid 0: {"prefix": "osd deep-scrub", "who": "3", "target": ["mon-mgr", ""]}) -- 0x7f8a0c0698c0 con 0x7f89f80514e0 2024-09-14T03:40:49.887 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.886+0000 7f8a08ff9640 1 -- 172.21.15.83:0/1934497398 <== mgr.14106 v1:172.21.15.83:6800/1803897559 1 ==== mgr_command_reply(tid 0: 0 instructed osd(s) 3 to deep-scrub) ==== 41+0+0 (unknown 3526570807 0 0) 0x7f8a0c0698c0 con 0x7f89f80514e0 2024-09-14T03:40:49.888 INFO:teuthology.orchestra.run.smithi083.stderr:instructed osd(s) 3 to deep-scrub 2024-09-14T03:40:49.889 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.887+0000 7f8a123e7640 1 -- 172.21.15.83:0/1934497398 >> v1:172.21.15.83:6800/1803897559 conn(0x7f89f80514e0 legacy=0x7f89f80539a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:49.889 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.887+0000 7f8a123e7640 1 -- 172.21.15.83:0/1934497398 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f8a0c098d40 msgr2=0x7f8a0c1c4a40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:49.889 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.887+0000 7f8a123e7640 1 --2- 172.21.15.83:0/1934497398 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f8a0c098d40 0x7f8a0c1c4a40 secure :-1 s=READY pgs=69 cs=0 l=1 rev1=1 crypto rx=0x7f89fc00e9c0 tx=0x7f89fc00ee80 comp rx=0 tx=0).stop 2024-09-14T03:40:49.890 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.887+0000 7f8a123e7640 1 -- 172.21.15.83:0/1934497398 shutdown_connections 2024-09-14T03:40:49.890 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.887+0000 7f8a123e7640 1 --2- 172.21.15.83:0/1934497398 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f8a0c12eff0 0x7f8a0c1c54c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:49.890 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.887+0000 7f8a123e7640 1 --2- 172.21.15.83:0/1934497398 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f8a0c099660 0x7f8a0c1c4f80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:49.890 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.887+0000 7f8a123e7640 1 --2- 172.21.15.83:0/1934497398 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f8a0c098d40 0x7f8a0c1c4a40 unknown :-1 s=CLOSED pgs=69 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:49.890 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.887+0000 7f8a123e7640 1 -- 172.21.15.83:0/1934497398 >> 172.21.15.83:0/1934497398 conn(0x7f8a0c0a1790 msgr2=0x7f8a0c128440 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:49.890 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.888+0000 7f8a123e7640 1 -- 172.21.15.83:0/1934497398 shutdown_connections 2024-09-14T03:40:49.890 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.888+0000 7f8a123e7640 1 -- 172.21.15.83:0/1934497398 wait complete. 2024-09-14T03:40:49.900 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-09-14T03:40:50.001 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.999+0000 7f6fddb28640 1 -- 172.21.15.83:0/582439838 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f6fd8131dc0 msgr2=0x7f6fd81341b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:50.001 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.999+0000 7f6fddb28640 1 --2- 172.21.15.83:0/582439838 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f6fd8131dc0 0x7f6fd81341b0 secure :-1 s=READY pgs=70 cs=0 l=1 rev1=1 crypto rx=0x7f6fc0009860 tx=0x7f6fc002a460 comp rx=0 tx=0).stop 2024-09-14T03:40:50.001 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.999+0000 7f6fddb28640 1 -- 172.21.15.83:0/582439838 shutdown_connections 2024-09-14T03:40:50.001 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.999+0000 7f6fddb28640 1 --2- 172.21.15.83:0/582439838 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f6fd81346f0 0x7f6fd8136ae0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:50.002 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.999+0000 7f6fddb28640 1 --2- 172.21.15.83:0/582439838 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f6fd8131dc0 0x7f6fd81341b0 unknown :-1 s=CLOSED pgs=70 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:50.002 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.999+0000 7f6fddb28640 1 -- 172.21.15.83:0/582439838 >> 172.21.15.83:0/582439838 conn(0x7f6fd80a1800 msgr2=0x7f6fd80a1c10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:50.002 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.999+0000 7f6fddb28640 1 -- 172.21.15.83:0/582439838 shutdown_connections 2024-09-14T03:40:50.002 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:49.999+0000 7f6fddb28640 1 -- 172.21.15.83:0/582439838 wait complete. 2024-09-14T03:40:50.002 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:50.000+0000 7f6fddb28640 1 Processor -- start 2024-09-14T03:40:50.002 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:50.000+0000 7f6fddb28640 1 -- start start 2024-09-14T03:40:50.005 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:50.000+0000 7f6fddb28640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f6fd8099080 0x7f6fd812dd70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:50.005 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:50.000+0000 7f6fddb28640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f6fd8131dc0 0x7f6fd812e2b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:50.005 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:50.000+0000 7f6fddb28640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f6fd81346f0 0x7f6fd812e7f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:40:50.005 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:50.000+0000 7f6fddb28640 1 -- --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7f6fd81377a0 con 0x7f6fd81346f0 2024-09-14T03:40:50.005 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:50.000+0000 7f6fddb28640 1 -- --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7f6fd8137620 con 0x7f6fd8131dc0 2024-09-14T03:40:50.005 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:50.000+0000 7f6fddb28640 1 -- --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7f6fd8137920 con 0x7f6fd8099080 2024-09-14T03:40:50.005 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:50.000+0000 7f6fceffd640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f6fd8131dc0 0x7f6fd812e2b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:50.005 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:50.000+0000 7f6fd77fe640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f6fd8099080 0x7f6fd812dd70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:50.005 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:50.000+0000 7f6fd7fff640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f6fd81346f0 0x7f6fd812e7f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:40:50.005 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:50.000+0000 7f6fceffd640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f6fd8131dc0 0x7f6fd812e2b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3301/0 says I am v2:172.21.15.83:54056/0 (socket says 172.21.15.83:54056) 2024-09-14T03:40:50.005 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:50.000+0000 7f6fceffd640 1 -- 172.21.15.83:0/3496841618 learned_addr learned my addr 172.21.15.83:0/3496841618 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:40:50.005 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:50.001+0000 7f6fd7fff640 1 -- 172.21.15.83:0/3496841618 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f6fd8099080 msgr2=0x7f6fd812dd70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:50.006 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:50.001+0000 7f6fd7fff640 1 --2- 172.21.15.83:0/3496841618 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f6fd8099080 0x7f6fd812dd70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:50.006 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:50.001+0000 7f6fd7fff640 1 -- 172.21.15.83:0/3496841618 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f6fd8131dc0 msgr2=0x7f6fd812e2b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:50.006 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:50.001+0000 7f6fd7fff640 1 --2- 172.21.15.83:0/3496841618 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f6fd8131dc0 0x7f6fd812e2b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:50.006 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:50.001+0000 7f6fd7fff640 1 -- 172.21.15.83:0/3496841618 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f6fd809ec60 con 0x7f6fd81346f0 2024-09-14T03:40:50.006 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:50.001+0000 7f6fceffd640 1 --2- 172.21.15.83:0/3496841618 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f6fd8131dc0 0x7f6fd812e2b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:40:50.006 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:50.001+0000 7f6fd7fff640 1 --2- 172.21.15.83:0/3496841618 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f6fd81346f0 0x7f6fd812e7f0 secure :-1 s=READY pgs=136 cs=0 l=1 rev1=1 crypto rx=0x7f6fd0005fd0 tx=0x7f6fd000c6a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:40:50.006 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:50.001+0000 7f6fd4ff9640 1 -- 172.21.15.83:0/3496841618 <== mon.0 v2:172.21.15.83:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6fd0010630 con 0x7f6fd81346f0 2024-09-14T03:40:50.006 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:50.001+0000 7f6fd4ff9640 1 -- 172.21.15.83:0/3496841618 <== mon.0 v2:172.21.15.83:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f6fd00107d0 con 0x7f6fd81346f0 2024-09-14T03:40:50.006 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:50.001+0000 7f6fddb28640 1 -- 172.21.15.83:0/3496841618 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f6fd812ed30 con 0x7f6fd81346f0 2024-09-14T03:40:50.006 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:50.001+0000 7f6fd4ff9640 1 -- 172.21.15.83:0/3496841618 <== mon.0 v2:172.21.15.83:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6fd0010c00 con 0x7f6fd81346f0 2024-09-14T03:40:50.006 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:50.001+0000 7f6fddb28640 1 -- 172.21.15.83:0/3496841618 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f6fd812f490 con 0x7f6fd81346f0 2024-09-14T03:40:50.006 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:50.002+0000 7f6fd4ff9640 1 -- 172.21.15.83:0/3496841618 <== mon.0 v2:172.21.15.83:3300/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7f6fd0021430 con 0x7f6fd81346f0 2024-09-14T03:40:50.006 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:50.002+0000 7f6fddb28640 1 -- 172.21.15.83:0/3496841618 --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f6fd8137dd0 con 0x7f6fd81346f0 2024-09-14T03:40:50.006 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:50.003+0000 7f6fd4ff9640 1 -- 172.21.15.83:0/3496841618 <== mon.0 v2:172.21.15.83:3300/0 5 ==== osd_map(33..33 src has 1..33) ==== 3027+0+0 (secure 0 0 0) 0x7f6fd0069e30 con 0x7f6fd81346f0 2024-09-14T03:40:50.007 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:50.006+0000 7f6fd4ff9640 1 -- 172.21.15.83:0/3496841618 <== mon.0 v2:172.21.15.83:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+157506 (secure 0 0 0) 0x7f6fd0068ee0 con 0x7f6fd81346f0 2024-09-14T03:40:50.154 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:50.153+0000 7f6fddb28640 1 -- 172.21.15.83:0/3496841618 --> v1:172.21.15.83:6800/1803897559 -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) -- 0x7f6fd812f750 con 0x7f6fb80514b0 2024-09-14T03:40:50.155 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:50.154+0000 7f6fd4ff9640 1 -- 172.21.15.83:0/3496841618 <== mgr.14106 v1:172.21.15.83:6800/1803897559 1 ==== mgr_command_reply(tid 0: 0 dumped all) ==== 18+0+31991 (unknown 2965378022 0 4067381145) 0x7f6fd812f750 con 0x7f6fb80514b0 2024-09-14T03:40:50.156 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:40:50.156 INFO:teuthology.orchestra.run.smithi083.stderr:dumped all 2024-09-14T03:40:50.157 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:50.156+0000 7f6fddb28640 1 -- 172.21.15.83:0/3496841618 >> v1:172.21.15.83:6800/1803897559 conn(0x7f6fb80514b0 legacy=0x7f6fb8053970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:50.157 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:50.156+0000 7f6fddb28640 1 -- 172.21.15.83:0/3496841618 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f6fd81346f0 msgr2=0x7f6fd812e7f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:40:50.157 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:50.156+0000 7f6fddb28640 1 --2- 172.21.15.83:0/3496841618 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f6fd81346f0 0x7f6fd812e7f0 secure :-1 s=READY pgs=136 cs=0 l=1 rev1=1 crypto rx=0x7f6fd0005fd0 tx=0x7f6fd000c6a0 comp rx=0 tx=0).stop 2024-09-14T03:40:50.158 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:50.156+0000 7f6fddb28640 1 -- 172.21.15.83:0/3496841618 shutdown_connections 2024-09-14T03:40:50.158 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:50.156+0000 7f6fddb28640 1 --2- 172.21.15.83:0/3496841618 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f6fd81346f0 0x7f6fd812e7f0 unknown :-1 s=CLOSED pgs=136 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:50.158 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:50.156+0000 7f6fddb28640 1 --2- 172.21.15.83:0/3496841618 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f6fd8131dc0 0x7f6fd812e2b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:50.158 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:50.156+0000 7f6fddb28640 1 --2- 172.21.15.83:0/3496841618 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f6fd8099080 0x7f6fd812dd70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:40:50.158 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:50.156+0000 7f6fddb28640 1 -- 172.21.15.83:0/3496841618 >> 172.21.15.83:0/3496841618 conn(0x7f6fd80a1800 msgr2=0x7f6fd80978b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:40:50.158 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:50.156+0000 7f6fddb28640 1 -- 172.21.15.83:0/3496841618 shutdown_connections 2024-09-14T03:40:50.158 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:40:50.156+0000 7f6fddb28640 1 -- 172.21.15.83:0/3496841618 wait complete. 2024-09-14T03:40:50.168 INFO:teuthology.orchestra.run.smithi083.stdout:{"pg_ready":true,"pg_map":{"version":30,"stamp":"2024-09-14T03:40:50.044374+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":166,"num_read_kb":267,"num_write":151,"num_write_kb":1896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":107,"ondisk_log_size":107,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":4,"num_per_pool_osds":4,"num_per_pool_omap_osds":4,"kb":377487360,"kb_used":112364,"kb_used_data":2256,"kb_used_omap":17,"kb_used_meta":109998,"kb_avail":377374996,"statfs":{"total":386547056640,"available":386431995904,"internally_reserved":0,"allocated":2310144,"data_stored":1564350,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":17731,"internal_metadata":112638653},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.931896"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":54,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-14T03:40:41.119404+0000","last_change":"2024-09-14T03:40:35.310297+0000","last_active":"2024-09-14T03:40:41.119404+0000","last_peered":"2024-09-14T03:40:41.119404+0000","last_clean":"2024-09-14T03:40:41.119404+0000","last_became_active":"2024-09-14T03:40:35.309980+0000","last_became_peered":"2024-09-14T03:40:35.309980+0000","last_unstale":"2024-09-14T03:40:41.119404+0000","last_undegraded":"2024-09-14T03:40:41.119404+0000","last_fullsized":"2024-09-14T03:40:41.119404+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_clean_scrub_stamp":"2024-09-14T03:39:00.791161+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:33:56.556777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00098140899999999993,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":44,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-14T03:40:41.120097+0000","last_change":"2024-09-14T03:40:33.102548+0000","last_active":"2024-09-14T03:40:41.120097+0000","last_peered":"2024-09-14T03:40:41.120097+0000","last_clean":"2024-09-14T03:40:41.120097+0000","last_became_active":"2024-09-14T03:40:33.102279+0000","last_became_peered":"2024-09-14T03:40:33.102279+0000","last_unstale":"2024-09-14T03:40:41.120097+0000","last_undegraded":"2024-09-14T03:40:41.120097+0000","last_fullsized":"2024-09-14T03:40:41.120097+0000","mapping_epoch":27,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":28,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_clean_scrub_stamp":"2024-09-14T03:39:00.791161+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:26:35.820454+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00045411099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":54,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-14T03:40:41.119610+0000","last_change":"2024-09-14T03:40:35.309898+0000","last_active":"2024-09-14T03:40:41.119610+0000","last_peered":"2024-09-14T03:40:41.119610+0000","last_clean":"2024-09-14T03:40:41.119610+0000","last_became_active":"2024-09-14T03:40:35.309659+0000","last_became_peered":"2024-09-14T03:40:35.309659+0000","last_unstale":"2024-09-14T03:40:41.119610+0000","last_undegraded":"2024-09-14T03:40:41.119610+0000","last_fullsized":"2024-09-14T03:40:41.119610+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_clean_scrub_stamp":"2024-09-14T03:39:00.791161+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:46:05.387098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00087291699999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":44,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-14T03:40:41.120130+0000","last_change":"2024-09-14T03:40:33.102548+0000","last_active":"2024-09-14T03:40:41.120130+0000","last_peered":"2024-09-14T03:40:41.120130+0000","last_clean":"2024-09-14T03:40:41.120130+0000","last_became_active":"2024-09-14T03:40:33.102255+0000","last_became_peered":"2024-09-14T03:40:33.102255+0000","last_unstale":"2024-09-14T03:40:41.120130+0000","last_undegraded":"2024-09-14T03:40:41.120130+0000","last_fullsized":"2024-09-14T03:40:41.120130+0000","mapping_epoch":27,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":28,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_clean_scrub_stamp":"2024-09-14T03:39:00.791161+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:46:51.929694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00048052199999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"18'2","reported_seq":52,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-14T03:40:41.119384+0000","last_change":"2024-09-14T03:40:35.110955+0000","last_active":"2024-09-14T03:40:41.119384+0000","last_peered":"2024-09-14T03:40:41.119384+0000","last_clean":"2024-09-14T03:40:41.119384+0000","last_became_active":"2024-09-14T03:40:35.110637+0000","last_became_peered":"2024-09-14T03:40:35.110637+0000","last_unstale":"2024-09-14T03:40:41.119384+0000","last_undegraded":"2024-09-14T03:40:41.119384+0000","last_fullsized":"2024-09-14T03:40:41.119384+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_clean_scrub_stamp":"2024-09-14T03:39:00.791161+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:35:22.417821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00135844,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":37,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-14T03:40:41.177732+0000","last_change":"2024-09-14T03:40:36.107870+0000","last_active":"2024-09-14T03:40:41.177732+0000","last_peered":"2024-09-14T03:40:41.177732+0000","last_clean":"2024-09-14T03:40:41.177732+0000","last_became_active":"2024-09-14T03:40:36.107387+0000","last_became_peered":"2024-09-14T03:40:36.107387+0000","last_unstale":"2024-09-14T03:40:41.177732+0000","last_undegraded":"2024-09-14T03:40:41.177732+0000","last_fullsized":"2024-09-14T03:40:41.177732+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_clean_scrub_stamp":"2024-09-14T03:39:00.791161+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:24:43.135910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046880499999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":50,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-14T03:40:41.119371+0000","last_change":"2024-09-14T03:40:35.105432+0000","last_active":"2024-09-14T03:40:41.119371+0000","last_peered":"2024-09-14T03:40:41.119371+0000","last_clean":"2024-09-14T03:40:41.119371+0000","last_became_active":"2024-09-14T03:40:35.105125+0000","last_became_peered":"2024-09-14T03:40:35.105125+0000","last_unstale":"2024-09-14T03:40:41.119371+0000","last_undegraded":"2024-09-14T03:40:41.119371+0000","last_fullsized":"2024-09-14T03:40:41.119371+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_clean_scrub_stamp":"2024-09-14T03:39:00.791161+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:06:08.387860+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000765802,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"16'1","reported_seq":48,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-14T03:40:41.119494+0000","last_change":"2024-09-14T03:40:36.107793+0000","last_active":"2024-09-14T03:40:41.119494+0000","last_peered":"2024-09-14T03:40:41.119494+0000","last_clean":"2024-09-14T03:40:41.119494+0000","last_became_active":"2024-09-14T03:40:36.107394+0000","last_became_peered":"2024-09-14T03:40:36.107394+0000","last_unstale":"2024-09-14T03:40:41.119494+0000","last_undegraded":"2024-09-14T03:40:41.119494+0000","last_fullsized":"2024-09-14T03:40:41.119494+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:39:00.791161+0000","last_clean_scrub_stamp":"2024-09-14T03:39:00.791161+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:34:57.278411+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00075472199999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"21'104","reported_seq":199,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-14T03:40:41.119989+0000","last_change":"2024-09-14T03:40:33.097410+0000","last_active":"2024-09-14T03:40:41.119989+0000","last_peered":"2024-09-14T03:40:41.119989+0000","last_clean":"2024-09-14T03:40:41.119989+0000","last_became_active":"2024-09-14T03:40:33.097097+0000","last_became_peered":"2024-09-14T03:40:33.097097+0000","last_unstale":"2024-09-14T03:40:41.119989+0000","last_undegraded":"2024-09-14T03:40:41.119989+0000","last_fullsized":"2024-09-14T03:40:41.119989+0000","mapping_epoch":27,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":28,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:38:57.773685+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:38:57.773685+0000","last_clean_scrub_stamp":"2024-09-14T03:38:57.773685+0000","objects_scrubbed":0,"log_size":104,"log_dups_size":0,"ondisk_log_size":104,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:08:55.427294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":166,"num_read_kb":267,"num_write":149,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":4},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":166,"num_read_kb":267,"num_write":149,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":104,"ondisk_log_size":104,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":3,"up_from":29,"seq":124554051589,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27820,"kb_used_data":276,"kb_used_omap":4,"kb_used_meta":27515,"kb_avail":94344020,"statfs":{"total":96636764160,"available":96608276480,"internally_reserved":0,"allocated":282624,"data_stored":95913,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4433,"internal_metadata":28176047},"hb_peers":[0,1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":30,"seq":128849018885,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27812,"kb_used_data":272,"kb_used_omap":4,"kb_used_meta":27515,"kb_avail":94344028,"statfs":{"total":96636764160,"available":96608284672,"internally_reserved":0,"allocated":278528,"data_stored":95894,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4432,"internal_metadata":28176048},"hb_peers":[0,1,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":27,"seq":115964116997,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":28336,"kb_used_data":856,"kb_used_omap":4,"kb_used_meta":27451,"kb_avail":94343504,"statfs":{"total":96636764160,"available":96607748096,"internally_reserved":0,"allocated":876544,"data_stored":686281,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4431,"internal_metadata":28110513},"hb_peers":[0,2,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":27,"seq":115964116997,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":28396,"kb_used_data":852,"kb_used_omap":4,"kb_used_meta":27515,"kb_avail":94343444,"statfs":{"total":96636764160,"available":96607686656,"internally_reserved":0,"allocated":872448,"data_stored":686262,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4435,"internal_metadata":28176045},"hb_peers":[1,2,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T03:40:50.172 INFO:tasks.ceph:pgid 2.7 last_scrub_stamp 2024-09-14T03:39:00.791161+0000 time.struct_time(tm_year=2024, tm_mon=9, tm_mday=14, tm_hour=3, tm_min=39, tm_sec=0, tm_wday=5, tm_yday=258, tm_isdst=-1) <= time.struct_time(tm_year=2024, tm_mon=9, tm_mday=14, tm_hour=3, tm_min=40, tm_sec=47, tm_wday=5, tm_yday=258, tm_isdst=0) 2024-09-14T03:40:50.172 INFO:tasks.ceph:pgid 2.6 last_scrub_stamp 2024-09-14T03:39:00.791161+0000 time.struct_time(tm_year=2024, tm_mon=9, tm_mday=14, tm_hour=3, tm_min=39, tm_sec=0, tm_wday=5, tm_yday=258, tm_isdst=-1) <= time.struct_time(tm_year=2024, tm_mon=9, tm_mday=14, tm_hour=3, tm_min=40, tm_sec=47, tm_wday=5, tm_yday=258, tm_isdst=0) 2024-09-14T03:40:50.172 INFO:tasks.ceph:pgid 2.5 last_scrub_stamp 2024-09-14T03:39:00.791161+0000 time.struct_time(tm_year=2024, tm_mon=9, tm_mday=14, tm_hour=3, tm_min=39, tm_sec=0, tm_wday=5, tm_yday=258, tm_isdst=-1) <= time.struct_time(tm_year=2024, tm_mon=9, tm_mday=14, tm_hour=3, tm_min=40, tm_sec=47, tm_wday=5, tm_yday=258, tm_isdst=0) 2024-09-14T03:40:50.172 INFO:tasks.ceph:pgid 2.4 last_scrub_stamp 2024-09-14T03:39:00.791161+0000 time.struct_time(tm_year=2024, tm_mon=9, tm_mday=14, tm_hour=3, tm_min=39, tm_sec=0, tm_wday=5, tm_yday=258, tm_isdst=-1) <= time.struct_time(tm_year=2024, tm_mon=9, tm_mday=14, tm_hour=3, tm_min=40, tm_sec=47, tm_wday=5, tm_yday=258, tm_isdst=0) 2024-09-14T03:40:50.172 INFO:tasks.ceph:pgid 2.2 last_scrub_stamp 2024-09-14T03:39:00.791161+0000 time.struct_time(tm_year=2024, tm_mon=9, tm_mday=14, tm_hour=3, tm_min=39, tm_sec=0, tm_wday=5, tm_yday=258, tm_isdst=-1) <= time.struct_time(tm_year=2024, tm_mon=9, tm_mday=14, tm_hour=3, tm_min=40, tm_sec=47, tm_wday=5, tm_yday=258, tm_isdst=0) 2024-09-14T03:40:50.172 INFO:tasks.ceph:pgid 2.1 last_scrub_stamp 2024-09-14T03:39:00.791161+0000 time.struct_time(tm_year=2024, tm_mon=9, tm_mday=14, tm_hour=3, tm_min=39, tm_sec=0, tm_wday=5, tm_yday=258, tm_isdst=-1) <= time.struct_time(tm_year=2024, tm_mon=9, tm_mday=14, tm_hour=3, tm_min=40, tm_sec=47, tm_wday=5, tm_yday=258, tm_isdst=0) 2024-09-14T03:40:50.172 INFO:tasks.ceph:pgid 2.0 last_scrub_stamp 2024-09-14T03:39:00.791161+0000 time.struct_time(tm_year=2024, tm_mon=9, tm_mday=14, tm_hour=3, tm_min=39, tm_sec=0, tm_wday=5, tm_yday=258, tm_isdst=-1) <= time.struct_time(tm_year=2024, tm_mon=9, tm_mday=14, tm_hour=3, tm_min=40, tm_sec=47, tm_wday=5, tm_yday=258, tm_isdst=0) 2024-09-14T03:40:50.172 INFO:tasks.ceph:pgid 2.3 last_scrub_stamp 2024-09-14T03:39:00.791161+0000 time.struct_time(tm_year=2024, tm_mon=9, tm_mday=14, tm_hour=3, tm_min=39, tm_sec=0, tm_wday=5, tm_yday=258, tm_isdst=-1) <= time.struct_time(tm_year=2024, tm_mon=9, tm_mday=14, tm_hour=3, tm_min=40, tm_sec=47, tm_wday=5, tm_yday=258, tm_isdst=0) 2024-09-14T03:40:50.173 INFO:tasks.ceph:pgid 1.0 last_scrub_stamp 2024-09-14T03:38:57.773685+0000 time.struct_time(tm_year=2024, tm_mon=9, tm_mday=14, tm_hour=3, tm_min=38, tm_sec=57, tm_wday=5, tm_yday=258, tm_isdst=-1) <= time.struct_time(tm_year=2024, tm_mon=9, tm_mday=14, tm_hour=3, tm_min=40, tm_sec=47, tm_wday=5, tm_yday=258, tm_isdst=0) 2024-09-14T03:40:50.173 INFO:tasks.ceph:Still waiting for all pgs to be scrubbed. 2024-09-14T03:41:10.174 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-09-14T03:41:10.288 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:41:10.286+0000 7f258268c640 1 -- 172.21.15.83:0/2934594554 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f257c12cc30 msgr2=0x7f257c12d010 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:41:10.288 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:41:10.286+0000 7f258268c640 1 --2- 172.21.15.83:0/2934594554 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f257c12cc30 0x7f257c12d010 secure :-1 s=READY pgs=71 cs=0 l=1 rev1=1 crypto rx=0x7f256c005fd0 tx=0x7f256c02a410 comp rx=0 tx=0).stop 2024-09-14T03:41:10.288 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:41:10.286+0000 7f258268c640 1 -- 172.21.15.83:0/2934594554 shutdown_connections 2024-09-14T03:41:10.288 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:41:10.286+0000 7f258268c640 1 --2- 172.21.15.83:0/2934594554 >> v2:172.21.15.83:3300/0 conn(0x7f257c094f10 0x7f257c133c00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:41:10.288 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:41:10.286+0000 7f258268c640 1 --2- 172.21.15.83:0/2934594554 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f257c0945d0 0x7f257c0949d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:41:10.288 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:41:10.286+0000 7f258268c640 1 --2- 172.21.15.83:0/2934594554 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f257c12cc30 0x7f257c12d010 unknown :-1 s=CLOSED pgs=71 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:41:10.288 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:41:10.286+0000 7f258268c640 1 -- 172.21.15.83:0/2934594554 >> 172.21.15.83:0/2934594554 conn(0x7f257c09a2e0 msgr2=0x7f257c097ad0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:41:10.288 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:41:10.286+0000 7f258268c640 1 -- 172.21.15.83:0/2934594554 shutdown_connections 2024-09-14T03:41:10.289 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:41:10.286+0000 7f258268c640 1 -- 172.21.15.83:0/2934594554 wait complete. 2024-09-14T03:41:10.291 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:41:10.287+0000 7f258268c640 1 Processor -- start 2024-09-14T03:41:10.291 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:41:10.287+0000 7f258268c640 1 -- start start 2024-09-14T03:41:10.291 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:41:10.287+0000 7f258268c640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f257c0945d0 0x7f257c122190 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:41:10.291 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:41:10.287+0000 7f258268c640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f257c094f10 0x7f257c1226d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:41:10.291 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:41:10.288+0000 7f258268c640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f257c12cc30 0x7f257c122c10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:41:10.291 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:41:10.288+0000 7f258268c640 1 -- --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7f257c134610 con 0x7f257c0945d0 2024-09-14T03:41:10.291 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:41:10.288+0000 7f258268c640 1 -- --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7f257c134490 con 0x7f257c12cc30 2024-09-14T03:41:10.291 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:41:10.288+0000 7f258268c640 1 -- --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7f257c134790 con 0x7f257c094f10 2024-09-14T03:41:10.291 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:41:10.288+0000 7f257bfff640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f257c0945d0 0x7f257c122190 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:41:10.291 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:41:10.288+0000 7f257b7fe640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f257c094f10 0x7f257c1226d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:41:10.292 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:41:10.288+0000 7f2580c29640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f257c12cc30 0x7f257c122c10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:41:10.292 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:41:10.288+0000 7f257bfff640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f257c0945d0 0x7f257c122190 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3300/0 says I am v2:172.21.15.83:41958/0 (socket says 172.21.15.83:41958) 2024-09-14T03:41:10.292 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:41:10.288+0000 7f257bfff640 1 -- 172.21.15.83:0/3152223150 learned_addr learned my addr 172.21.15.83:0/3152223150 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:41:10.292 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:41:10.288+0000 7f257b7fe640 1 -- 172.21.15.83:0/3152223150 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f257c12cc30 msgr2=0x7f257c122c10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:41:10.292 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:41:10.288+0000 7f257b7fe640 1 --2- 172.21.15.83:0/3152223150 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f257c12cc30 0x7f257c122c10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:41:10.292 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:41:10.288+0000 7f257b7fe640 1 -- 172.21.15.83:0/3152223150 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f257c0945d0 msgr2=0x7f257c122190 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:41:10.292 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:41:10.288+0000 7f257b7fe640 1 --2- 172.21.15.83:0/3152223150 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f257c0945d0 0x7f257c122190 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:41:10.292 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:41:10.288+0000 7f257b7fe640 1 -- 172.21.15.83:0/3152223150 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f257c131060 con 0x7f257c094f10 2024-09-14T03:41:10.292 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:41:10.288+0000 7f257b7fe640 1 --2- 172.21.15.83:0/3152223150 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f257c094f10 0x7f257c1226d0 secure :-1 s=READY pgs=70 cs=0 l=1 rev1=1 crypto rx=0x7f257000e9c0 tx=0x7f257000ee80 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:41:10.292 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:41:10.288+0000 7f2578ff9640 1 -- 172.21.15.83:0/3152223150 <== mon.2 v2:172.21.15.83:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f257000ccc0 con 0x7f257c094f10 2024-09-14T03:41:10.292 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:41:10.288+0000 7f2578ff9640 1 -- 172.21.15.83:0/3152223150 <== mon.2 v2:172.21.15.83:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f2570004020 con 0x7f257c094f10 2024-09-14T03:41:10.292 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:41:10.288+0000 7f258268c640 1 -- 172.21.15.83:0/3152223150 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f257c1309a0 con 0x7f257c094f10 2024-09-14T03:41:10.292 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:41:10.288+0000 7f2578ff9640 1 -- 172.21.15.83:0/3152223150 <== mon.2 v2:172.21.15.83:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2570004470 con 0x7f257c094f10 2024-09-14T03:41:10.292 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:41:10.288+0000 7f258268c640 1 -- 172.21.15.83:0/3152223150 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7f257c1241b0 con 0x7f257c094f10 2024-09-14T03:41:10.293 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:41:10.289+0000 7f258268c640 1 -- 172.21.15.83:0/3152223150 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f257c12bc70 con 0x7f257c094f10 2024-09-14T03:41:10.293 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:41:10.289+0000 7f2578ff9640 1 -- 172.21.15.83:0/3152223150 <== mon.2 v2:172.21.15.83:3302/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7f25700047a0 con 0x7f257c094f10 2024-09-14T03:41:10.293 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:41:10.290+0000 7f2578ff9640 1 -- 172.21.15.83:0/3152223150 <== mon.2 v2:172.21.15.83:3302/0 5 ==== osd_map(33..33 src has 1..33) ==== 3027+0+0 (secure 0 0 0) 0x7f25700695a0 con 0x7f257c094f10 2024-09-14T03:41:10.295 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:41:10.293+0000 7f2578ff9640 1 -- 172.21.15.83:0/3152223150 <== mon.2 v2:172.21.15.83:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+157506 (secure 0 0 0) 0x7f257c12bc70 con 0x7f257c094f10 2024-09-14T03:41:10.440 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:41:10.439+0000 7f258268c640 1 -- 172.21.15.83:0/3152223150 --> v1:172.21.15.83:6800/1803897559 -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) -- 0x7f257c096bd0 con 0x7f25680514e0 2024-09-14T03:41:10.441 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:41:10.440+0000 7f2578ff9640 1 -- 172.21.15.83:0/3152223150 <== mgr.14106 v1:172.21.15.83:6800/1803897559 1 ==== mgr_command_reply(tid 0: 0 dumped all) ==== 18+0+32002 (unknown 2965378022 0 1100388018) 0x7f257c096bd0 con 0x7f25680514e0 2024-09-14T03:41:10.441 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-14T03:41:10.442 INFO:teuthology.orchestra.run.smithi083.stderr:dumped all 2024-09-14T03:41:10.443 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:41:10.441+0000 7f258268c640 1 -- 172.21.15.83:0/3152223150 >> v1:172.21.15.83:6800/1803897559 conn(0x7f25680514e0 legacy=0x7f25680539a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:41:10.443 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:41:10.441+0000 7f258268c640 1 -- 172.21.15.83:0/3152223150 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f257c094f10 msgr2=0x7f257c1226d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:41:10.443 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:41:10.441+0000 7f258268c640 1 --2- 172.21.15.83:0/3152223150 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f257c094f10 0x7f257c1226d0 secure :-1 s=READY pgs=70 cs=0 l=1 rev1=1 crypto rx=0x7f257000e9c0 tx=0x7f257000ee80 comp rx=0 tx=0).stop 2024-09-14T03:41:10.443 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:41:10.441+0000 7f258268c640 1 -- 172.21.15.83:0/3152223150 shutdown_connections 2024-09-14T03:41:10.443 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:41:10.441+0000 7f258268c640 1 --2- 172.21.15.83:0/3152223150 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f257c12cc30 0x7f257c122c10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:41:10.443 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:41:10.441+0000 7f258268c640 1 --2- 172.21.15.83:0/3152223150 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f257c094f10 0x7f257c1226d0 unknown :-1 s=CLOSED pgs=70 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:41:10.443 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:41:10.441+0000 7f258268c640 1 --2- 172.21.15.83:0/3152223150 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f257c0945d0 0x7f257c122190 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:41:10.444 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:41:10.441+0000 7f258268c640 1 -- 172.21.15.83:0/3152223150 >> 172.21.15.83:0/3152223150 conn(0x7f257c09a2e0 msgr2=0x7f257c09afd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:41:10.444 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:41:10.441+0000 7f258268c640 1 -- 172.21.15.83:0/3152223150 shutdown_connections 2024-09-14T03:41:10.444 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:41:10.442+0000 7f258268c640 1 -- 172.21.15.83:0/3152223150 wait complete. 2024-09-14T03:41:10.454 INFO:teuthology.orchestra.run.smithi083.stdout:{"pg_ready":true,"pg_map":{"version":40,"stamp":"2024-09-14T03:41:10.048347+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":166,"num_read_kb":267,"num_write":151,"num_write_kb":1896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":110,"ondisk_log_size":110,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":4,"num_per_pool_osds":4,"num_per_pool_omap_osds":4,"kb":377487360,"kb_used":112336,"kb_used_data":2256,"kb_used_omap":17,"kb_used_meta":109998,"kb_avail":377375024,"statfs":{"total":386547056640,"available":386432024576,"internally_reserved":0,"allocated":2310144,"data_stored":1564350,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":17731,"internal_metadata":112638653},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002514"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":62,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-14T03:40:52.086948+0000","last_change":"2024-09-14T03:40:52.086948+0000","last_active":"2024-09-14T03:40:52.086948+0000","last_peered":"2024-09-14T03:40:52.086948+0000","last_clean":"2024-09-14T03:40:52.086948+0000","last_became_active":"2024-09-14T03:40:35.309980+0000","last_became_peered":"2024-09-14T03:40:35.309980+0000","last_unstale":"2024-09-14T03:40:52.086948+0000","last_undegraded":"2024-09-14T03:40:52.086948+0000","last_fullsized":"2024-09-14T03:40:52.086948+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:40:52.086829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:40:52.086829+0000","last_clean_scrub_stamp":"2024-09-14T03:40:52.086829+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:26:35.051131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00098140899999999993,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":52,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-14T03:40:51.710640+0000","last_change":"2024-09-14T03:40:51.710640+0000","last_active":"2024-09-14T03:40:51.710640+0000","last_peered":"2024-09-14T03:40:51.710640+0000","last_clean":"2024-09-14T03:40:51.710640+0000","last_became_active":"2024-09-14T03:40:33.102279+0000","last_became_peered":"2024-09-14T03:40:33.102279+0000","last_unstale":"2024-09-14T03:40:51.710640+0000","last_undegraded":"2024-09-14T03:40:51.710640+0000","last_fullsized":"2024-09-14T03:40:51.710640+0000","mapping_epoch":27,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":28,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:40:51.710524+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:40:51.710524+0000","last_clean_scrub_stamp":"2024-09-14T03:40:51.710524+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:04:57.596186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00045411099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":62,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-14T03:40:54.123954+0000","last_change":"2024-09-14T03:40:54.123954+0000","last_active":"2024-09-14T03:40:54.123954+0000","last_peered":"2024-09-14T03:40:54.123954+0000","last_clean":"2024-09-14T03:40:54.123954+0000","last_became_active":"2024-09-14T03:40:35.309659+0000","last_became_peered":"2024-09-14T03:40:35.309659+0000","last_unstale":"2024-09-14T03:40:54.123954+0000","last_undegraded":"2024-09-14T03:40:54.123954+0000","last_fullsized":"2024-09-14T03:40:54.123954+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:40:54.123849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:40:54.123849+0000","last_clean_scrub_stamp":"2024-09-14T03:40:54.123849+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:06:15.603103+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00087291699999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":52,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-14T03:40:50.746824+0000","last_change":"2024-09-14T03:40:50.746824+0000","last_active":"2024-09-14T03:40:50.746824+0000","last_peered":"2024-09-14T03:40:50.746824+0000","last_clean":"2024-09-14T03:40:50.746824+0000","last_became_active":"2024-09-14T03:40:33.102255+0000","last_became_peered":"2024-09-14T03:40:33.102255+0000","last_unstale":"2024-09-14T03:40:50.746824+0000","last_undegraded":"2024-09-14T03:40:50.746824+0000","last_fullsized":"2024-09-14T03:40:50.746824+0000","mapping_epoch":27,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":28,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:40:50.746740+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:40:50.746740+0000","last_clean_scrub_stamp":"2024-09-14T03:40:50.746740+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:10:05.433200+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00048052199999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"33'3","reported_seq":61,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-14T03:40:51.090029+0000","last_change":"2024-09-14T03:40:51.090029+0000","last_active":"2024-09-14T03:40:51.090029+0000","last_peered":"2024-09-14T03:40:51.090029+0000","last_clean":"2024-09-14T03:40:51.090029+0000","last_became_active":"2024-09-14T03:40:35.110637+0000","last_became_peered":"2024-09-14T03:40:35.110637+0000","last_unstale":"2024-09-14T03:40:51.090029+0000","last_undegraded":"2024-09-14T03:40:51.090029+0000","last_fullsized":"2024-09-14T03:40:51.090029+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"33'3","last_scrub_stamp":"2024-09-14T03:40:51.089936+0000","last_deep_scrub":"33'3","last_deep_scrub_stamp":"2024-09-14T03:40:51.089936+0000","last_clean_scrub_stamp":"2024-09-14T03:40:51.089936+0000","objects_scrubbed":1,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:44:45.801162+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0.00135844,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":45,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-14T03:40:50.138834+0000","last_change":"2024-09-14T03:40:50.138834+0000","last_active":"2024-09-14T03:40:50.138834+0000","last_peered":"2024-09-14T03:40:50.138834+0000","last_clean":"2024-09-14T03:40:50.138834+0000","last_became_active":"2024-09-14T03:40:36.107387+0000","last_became_peered":"2024-09-14T03:40:36.107387+0000","last_unstale":"2024-09-14T03:40:50.138834+0000","last_undegraded":"2024-09-14T03:40:50.138834+0000","last_fullsized":"2024-09-14T03:40:50.138834+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:40:50.138759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:40:50.138759+0000","last_clean_scrub_stamp":"2024-09-14T03:40:50.138759+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:52:37.085661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046880499999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":58,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-14T03:40:50.113367+0000","last_change":"2024-09-14T03:40:50.113367+0000","last_active":"2024-09-14T03:40:50.113367+0000","last_peered":"2024-09-14T03:40:50.113367+0000","last_clean":"2024-09-14T03:40:50.113367+0000","last_became_active":"2024-09-14T03:40:35.105125+0000","last_became_peered":"2024-09-14T03:40:35.105125+0000","last_unstale":"2024-09-14T03:40:50.113367+0000","last_undegraded":"2024-09-14T03:40:50.113367+0000","last_fullsized":"2024-09-14T03:40:50.113367+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:40:50.113298+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:40:50.113298+0000","last_clean_scrub_stamp":"2024-09-14T03:40:50.113298+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:16:38.008519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000765802,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"33'2","reported_seq":57,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-14T03:40:53.081699+0000","last_change":"2024-09-14T03:40:53.081699+0000","last_active":"2024-09-14T03:40:53.081699+0000","last_peered":"2024-09-14T03:40:53.081699+0000","last_clean":"2024-09-14T03:40:53.081699+0000","last_became_active":"2024-09-14T03:40:36.107394+0000","last_became_peered":"2024-09-14T03:40:36.107394+0000","last_unstale":"2024-09-14T03:40:53.081699+0000","last_undegraded":"2024-09-14T03:40:53.081699+0000","last_fullsized":"2024-09-14T03:40:53.081699+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"33'2","last_scrub_stamp":"2024-09-14T03:40:53.081610+0000","last_deep_scrub":"33'2","last_deep_scrub_stamp":"2024-09-14T03:40:53.081610+0000","last_clean_scrub_stamp":"2024-09-14T03:40:53.081610+0000","objects_scrubbed":1,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:40:38.274101+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0.00075472199999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"33'105","reported_seq":208,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-14T03:40:49.765981+0000","last_change":"2024-09-14T03:40:49.765981+0000","last_active":"2024-09-14T03:40:49.765981+0000","last_peered":"2024-09-14T03:40:49.765981+0000","last_clean":"2024-09-14T03:40:49.765981+0000","last_became_active":"2024-09-14T03:40:33.097097+0000","last_became_peered":"2024-09-14T03:40:33.097097+0000","last_unstale":"2024-09-14T03:40:49.765981+0000","last_undegraded":"2024-09-14T03:40:49.765981+0000","last_fullsized":"2024-09-14T03:40:49.765981+0000","mapping_epoch":27,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":28,"parent":"0.0","parent_split_bits":0,"last_scrub":"33'105","last_scrub_stamp":"2024-09-14T03:40:49.765905+0000","last_deep_scrub":"33'105","last_deep_scrub_stamp":"2024-09-14T03:40:49.765905+0000","last_clean_scrub_stamp":"2024-09-14T03:40:49.765905+0000","objects_scrubbed":2,"log_size":105,"log_dups_size":0,"ondisk_log_size":105,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:52:30.502358+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":166,"num_read_kb":267,"num_write":149,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":5,"ondisk_log_size":5,"up":16,"acting":16,"num_store_stats":4},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":166,"num_read_kb":267,"num_write":149,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":105,"ondisk_log_size":105,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":3,"up_from":29,"seq":124554051593,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27812,"kb_used_data":276,"kb_used_omap":4,"kb_used_meta":27515,"kb_avail":94344028,"statfs":{"total":96636764160,"available":96608284672,"internally_reserved":0,"allocated":282624,"data_stored":95913,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4433,"internal_metadata":28176047},"hb_peers":[0,1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":30,"seq":128849018889,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27808,"kb_used_data":272,"kb_used_omap":4,"kb_used_meta":27515,"kb_avail":94344032,"statfs":{"total":96636764160,"available":96608288768,"internally_reserved":0,"allocated":278528,"data_stored":95894,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4432,"internal_metadata":28176048},"hb_peers":[0,1,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":27,"seq":115964117001,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":28328,"kb_used_data":856,"kb_used_omap":4,"kb_used_meta":27451,"kb_avail":94343512,"statfs":{"total":96636764160,"available":96607756288,"internally_reserved":0,"allocated":876544,"data_stored":686281,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4431,"internal_metadata":28110513},"hb_peers":[0,2,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":27,"seq":115964117001,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":28388,"kb_used_data":852,"kb_used_omap":4,"kb_used_meta":27515,"kb_avail":94343452,"statfs":{"total":96636764160,"available":96607694848,"internally_reserved":0,"allocated":872448,"data_stored":686262,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4435,"internal_metadata":28176045},"hb_peers":[1,2,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T03:41:10.455 DEBUG:teuthology.orchestra.run.smithi083:> sudo ceph --cluster ceph config set global mon_health_to_clog false 2024-09-14T03:41:10.555 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:41:10.553+0000 7f2428273640 1 -- 172.21.15.83:0/1880957631 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f24200962d0 msgr2=0x7f24200966b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:41:10.555 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:41:10.553+0000 7f2428273640 1 --2- 172.21.15.83:0/1880957631 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f24200962d0 0x7f24200966b0 secure :-1 s=READY pgs=71 cs=0 l=1 rev1=1 crypto rx=0x7f241c009830 tx=0x7f241c02a3f0 comp rx=0 tx=0).stop 2024-09-14T03:41:10.555 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:41:10.553+0000 7f2428273640 1 -- 172.21.15.83:0/1880957631 shutdown_connections 2024-09-14T03:41:10.555 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:41:10.553+0000 7f2428273640 1 --2- 172.21.15.83:0/1880957631 >> v2:172.21.15.83:3300/0 conn(0x7f2420093700 0x7f2420133ce0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:41:10.555 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:41:10.553+0000 7f2428273640 1 --2- 172.21.15.83:0/1880957631 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f24200962d0 0x7f24200966b0 unknown :-1 s=CLOSED pgs=71 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:41:10.555 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:41:10.553+0000 7f2428273640 1 -- 172.21.15.83:0/1880957631 >> 172.21.15.83:0/1880957631 conn(0x7f242009b2c0 msgr2=0x7f242009b6d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:41:10.555 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:41:10.553+0000 7f2428273640 1 -- 172.21.15.83:0/1880957631 shutdown_connections 2024-09-14T03:41:10.556 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:41:10.553+0000 7f2428273640 1 -- 172.21.15.83:0/1880957631 wait complete. 2024-09-14T03:41:10.558 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:41:10.554+0000 7f2428273640 1 Processor -- start 2024-09-14T03:41:10.558 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:41:10.554+0000 7f2428273640 1 -- start start 2024-09-14T03:41:10.558 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:41:10.554+0000 7f2428273640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f2420093700 0x7f2420133340 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:41:10.558 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:41:10.554+0000 7f2428273640 1 --2- >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f24200962d0 0x7f2420133880 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:41:10.558 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:41:10.554+0000 7f2428273640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f2420096bf0 0x7f2420129600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-14T03:41:10.558 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:41:10.554+0000 7f2428273640 1 -- --> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] -- mon_getmap magic: 0 -- 0x7f2420134600 con 0x7f2420093700 2024-09-14T03:41:10.558 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:41:10.554+0000 7f2428273640 1 -- --> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] -- mon_getmap magic: 0 -- 0x7f2420134480 con 0x7f24200962d0 2024-09-14T03:41:10.558 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:41:10.554+0000 7f2428273640 1 -- --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_getmap magic: 0 -- 0x7f2420134780 con 0x7f2420096bf0 2024-09-14T03:41:10.559 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:41:10.555+0000 7f242600f640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f2420093700 0x7f2420133340 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:41:10.559 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:41:10.555+0000 7f242600f640 1 --2- >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f2420093700 0x7f2420133340 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.83:3300/0 says I am v2:172.21.15.83:41968/0 (socket says 172.21.15.83:41968) 2024-09-14T03:41:10.559 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:41:10.555+0000 7f2426810640 1 --2- >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f2420096bf0 0x7f2420129600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:41:10.559 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:41:10.555+0000 7f242600f640 1 -- 172.21.15.83:0/3162989128 learned_addr learned my addr 172.21.15.83:0/3162989128 (peer_addr_for_me v2:172.21.15.83:0/0) 2024-09-14T03:41:10.559 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:41:10.555+0000 7f242580e640 1 --2- 172.21.15.83:0/3162989128 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f24200962d0 0x7f2420133880 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-14T03:41:10.559 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:41:10.555+0000 7f2426810640 1 -- 172.21.15.83:0/3162989128 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f24200962d0 msgr2=0x7f2420133880 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:41:10.559 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:41:10.555+0000 7f2426810640 1 --2- 172.21.15.83:0/3162989128 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f24200962d0 0x7f2420133880 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:41:10.559 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:41:10.555+0000 7f2426810640 1 -- 172.21.15.83:0/3162989128 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f2420093700 msgr2=0x7f2420133340 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:41:10.559 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:41:10.555+0000 7f2426810640 1 --2- 172.21.15.83:0/3162989128 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f2420093700 0x7f2420133340 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:41:10.559 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:41:10.555+0000 7f2426810640 1 -- 172.21.15.83:0/3162989128 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f2420127ab0 con 0x7f2420096bf0 2024-09-14T03:41:10.559 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:41:10.555+0000 7f242580e640 1 --2- 172.21.15.83:0/3162989128 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f24200962d0 0x7f2420133880 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:41:10.559 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:41:10.555+0000 7f242600f640 1 --2- 172.21.15.83:0/3162989128 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f2420093700 0x7f2420133340 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-14T03:41:10.559 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:41:10.555+0000 7f2426810640 1 --2- 172.21.15.83:0/3162989128 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f2420096bf0 0x7f2420129600 secure :-1 s=READY pgs=72 cs=0 l=1 rev1=1 crypto rx=0x7f241800ec50 tx=0x7f241800c6a0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-14T03:41:10.559 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:41:10.555+0000 7f2416ffd640 1 -- 172.21.15.83:0/3162989128 <== mon.2 v2:172.21.15.83:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2418003b70 con 0x7f2420096bf0 2024-09-14T03:41:10.560 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:41:10.555+0000 7f2416ffd640 1 -- 172.21.15.83:0/3162989128 <== mon.2 v2:172.21.15.83:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f2418003d10 con 0x7f2420096bf0 2024-09-14T03:41:10.560 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:41:10.555+0000 7f2428273640 1 -- 172.21.15.83:0/3162989128 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f2420126f00 con 0x7f2420096bf0 2024-09-14T03:41:10.560 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:41:10.555+0000 7f2416ffd640 1 -- 172.21.15.83:0/3162989128 <== mon.2 v2:172.21.15.83:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2418004280 con 0x7f2420096bf0 2024-09-14T03:41:10.560 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:41:10.555+0000 7f2428273640 1 -- 172.21.15.83:0/3162989128 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7f2420129d20 con 0x7f2420096bf0 2024-09-14T03:41:10.560 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:41:10.556+0000 7f2428273640 1 -- 172.21.15.83:0/3162989128 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f24200966b0 con 0x7f2420096bf0 2024-09-14T03:41:10.560 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:41:10.556+0000 7f2416ffd640 1 -- 172.21.15.83:0/3162989128 <== mon.2 v2:172.21.15.83:3302/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7f2418004420 con 0x7f2420096bf0 2024-09-14T03:41:10.560 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:41:10.557+0000 7f2416ffd640 1 -- 172.21.15.83:0/3162989128 <== mon.2 v2:172.21.15.83:3302/0 5 ==== osd_map(33..33 src has 1..33) ==== 3027+0+0 (secure 0 0 0) 0x7f2418069c20 con 0x7f2420096bf0 2024-09-14T03:41:10.561 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:41:10.560+0000 7f2416ffd640 1 -- 172.21.15.83:0/3162989128 <== mon.2 v2:172.21.15.83:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+157506 (secure 0 0 0) 0x7f2418068cd0 con 0x7f2420096bf0 2024-09-14T03:41:10.712 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:41:10.711+0000 7f2428273640 1 -- 172.21.15.83:0/3162989128 --> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] -- mon_command([{prefix=config set, name=mon_health_to_clog}] v 0) -- 0x7f2420069aa0 con 0x7f2420096bf0 2024-09-14T03:41:10.718 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:41:10.717+0000 7f2416ffd640 1 -- 172.21.15.83:0/3162989128 <== mon.2 v2:172.21.15.83:3302/0 7 ==== config(6 keys) ==== 172+0+0 (secure 0 0 0) 0x7f241806f020 con 0x7f2420096bf0 2024-09-14T03:41:10.718 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:41:10.717+0000 7f2416ffd640 1 -- 172.21.15.83:0/3162989128 <== mon.2 v2:172.21.15.83:3302/0 8 ==== mon_command_ack([{prefix=config set, name=mon_health_to_clog}]=0 v24)=0 v24) ==== 123+0+0 (secure 0 0 0) 0x7f2418004710 con 0x7f2420096bf0 2024-09-14T03:41:10.721 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:41:10.719+0000 7f2428273640 1 -- 172.21.15.83:0/3162989128 >> v1:172.21.15.83:6800/1803897559 conn(0x7f24040514b0 legacy=0x7f2404053970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:41:10.721 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:41:10.719+0000 7f2428273640 1 -- 172.21.15.83:0/3162989128 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f2420096bf0 msgr2=0x7f2420129600 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-14T03:41:10.721 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:41:10.719+0000 7f2428273640 1 --2- 172.21.15.83:0/3162989128 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f2420096bf0 0x7f2420129600 secure :-1 s=READY pgs=72 cs=0 l=1 rev1=1 crypto rx=0x7f241800ec50 tx=0x7f241800c6a0 comp rx=0 tx=0).stop 2024-09-14T03:41:10.721 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:41:10.719+0000 7f2428273640 1 -- 172.21.15.83:0/3162989128 shutdown_connections 2024-09-14T03:41:10.721 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:41:10.719+0000 7f2428273640 1 --2- 172.21.15.83:0/3162989128 >> [v2:172.21.15.83:3302/0,v1:172.21.15.83:6791/0] conn(0x7f2420096bf0 0x7f2420129600 unknown :-1 s=CLOSED pgs=72 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:41:10.721 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:41:10.719+0000 7f2428273640 1 --2- 172.21.15.83:0/3162989128 >> [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] conn(0x7f24200962d0 0x7f2420133880 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:41:10.721 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:41:10.719+0000 7f2428273640 1 --2- 172.21.15.83:0/3162989128 >> [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] conn(0x7f2420093700 0x7f2420133340 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-14T03:41:10.721 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:41:10.719+0000 7f2428273640 1 -- 172.21.15.83:0/3162989128 >> 172.21.15.83:0/3162989128 conn(0x7f242009b2c0 msgr2=0x7f2420071410 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-14T03:41:10.721 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:41:10.719+0000 7f2428273640 1 -- 172.21.15.83:0/3162989128 shutdown_connections 2024-09-14T03:41:10.721 INFO:teuthology.orchestra.run.smithi083.stderr:2024-09-14T03:41:10.719+0000 7f2428273640 1 -- 172.21.15.83:0/3162989128 wait complete. 2024-09-14T03:41:10.732 INFO:teuthology.misc:Shutting down mds daemons... 2024-09-14T03:41:10.732 INFO:teuthology.misc:Shutting down osd daemons... 2024-09-14T03:41:10.732 DEBUG:tasks.ceph.osd.0:waiting for process to exit 2024-09-14T03:41:10.732 INFO:teuthology.orchestra.run:waiting for 300 2024-09-14T03:41:10.759 INFO:tasks.ceph.osd.0:Stopped 2024-09-14T03:41:10.759 DEBUG:tasks.ceph.osd.1:waiting for process to exit 2024-09-14T03:41:10.759 INFO:teuthology.orchestra.run:waiting for 300 2024-09-14T03:41:10.835 INFO:tasks.ceph.osd.1:Stopped 2024-09-14T03:41:10.835 DEBUG:tasks.ceph.osd.2:waiting for process to exit 2024-09-14T03:41:10.835 INFO:teuthology.orchestra.run:waiting for 300 2024-09-14T03:41:10.908 INFO:tasks.ceph.osd.2:Stopped 2024-09-14T03:41:10.908 DEBUG:tasks.ceph.osd.3:waiting for process to exit 2024-09-14T03:41:10.908 INFO:teuthology.orchestra.run:waiting for 300 2024-09-14T03:41:10.979 INFO:tasks.ceph.osd.3:Stopped 2024-09-14T03:41:10.979 INFO:teuthology.misc:Shutting down mgr daemons... 2024-09-14T03:41:10.979 DEBUG:tasks.ceph.mgr.x:waiting for process to exit 2024-09-14T03:41:10.979 INFO:teuthology.orchestra.run:waiting for 300 2024-09-14T03:41:11.042 INFO:tasks.ceph.mgr.x:Stopped 2024-09-14T03:41:11.042 DEBUG:tasks.ceph.mgr.y:waiting for process to exit 2024-09-14T03:41:11.042 INFO:teuthology.orchestra.run:waiting for 300 2024-09-14T03:41:11.106 INFO:tasks.ceph.mgr.y:Stopped 2024-09-14T03:41:11.106 INFO:teuthology.misc:Shutting down mon daemons... 2024-09-14T03:41:11.107 DEBUG:tasks.ceph.mon.a:waiting for process to exit 2024-09-14T03:41:11.107 INFO:teuthology.orchestra.run:waiting for 300 2024-09-14T03:41:11.157 INFO:tasks.ceph.mon.a:Stopped 2024-09-14T03:41:11.158 DEBUG:tasks.ceph.mon.b:waiting for process to exit 2024-09-14T03:41:11.158 INFO:teuthology.orchestra.run:waiting for 300 2024-09-14T03:41:11.209 INFO:tasks.ceph.mon.b:Stopped 2024-09-14T03:41:11.210 DEBUG:tasks.ceph.mon.c:waiting for process to exit 2024-09-14T03:41:11.210 INFO:teuthology.orchestra.run:waiting for 300 2024-09-14T03:41:11.262 INFO:tasks.ceph.mon.c:Stopped 2024-09-14T03:41:11.262 INFO:tasks.ceph:Checking cluster log for badness... 2024-09-14T03:41:11.262 DEBUG:teuthology.orchestra.run.smithi083:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/ceph.log | egrep -v 'overall HEALTH_' | egrep -v '\(FS_' | egrep -v '\(MDS_' | egrep -v '\(OSD_' | egrep -v '\(MON_DOWN\)' | egrep -v '\(CACHE_POOL_' | egrep -v '\(POOL_' | egrep -v '\(MGR_DOWN\)' | egrep -v '\(PG_' | egrep -v '\(SMALLER_PGP_NUM\)' | egrep -v 'Monitor daemon marked osd' | egrep -v 'Behind on trimming' | egrep -v 'Manager daemon' | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | egrep -v FS_DEGRADED | egrep -v 'fs.*is degraded' | egrep -v 'filesystem is degraded' | egrep -v FS_INLINE_DATA_DEPRECATED | egrep -v FS_WITH_FAILED_MDS | egrep -v MDS_ALL_DOWN | egrep -v 'filesystem is offline' | egrep -v MDS_DAMAGE | egrep -v MDS_DEGRADED | egrep -v MDS_FAILED | egrep -v MDS_INSUFFICIENT_STANDBY | egrep -v 'insufficient standby MDS daemons available' | egrep -v MDS_UP_LESS_THAN_MAX | egrep -v 'online, but wants' | egrep -v 'filesystem is online with fewer MDS than max_mds' | egrep -v POOL_APP_NOT_ENABLED | egrep -v 'do not have an application enabled' | egrep -v 'overall HEALTH_' | egrep -v 'Replacing daemon' | egrep -v 'deprecated feature inline_data' | egrep -v 'overall HEALTH_' | egrep -v '\(OSD_DOWN\)' | egrep -v '\(OSD_' | egrep -v 'but it is still running' | egrep -v 'is not responding' | egrep -v PG_AVAILABILITY | egrep -v PG_DEGRADED | egrep -v 'Reduced data availability' | egrep -v 'Degraded data redundancy' | egrep -v 'scrub mismatch' | egrep -v ScrubResult | egrep -v 'wrongly marked' | egrep -v '\(POOL_APP_NOT_ENABLED\)' | egrep -v '\(SLOW_OPS\)' | egrep -v 'overall HEALTH_' | egrep -v '\(MON_MSGR2_NOT_ENABLED\)' | egrep -v 'slow request' | head -n 1 2024-09-14T03:41:11.342 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/ceph-0 on ubuntu@smithi083.front.sepia.ceph.com 2024-09-14T03:41:11.343 DEBUG:teuthology.orchestra.run.smithi083:> sync && sudo umount -f /var/lib/ceph/osd/ceph-0 2024-09-14T03:41:11.559 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/ceph-1 on ubuntu@smithi083.front.sepia.ceph.com 2024-09-14T03:41:11.559 DEBUG:teuthology.orchestra.run.smithi083:> sync && sudo umount -f /var/lib/ceph/osd/ceph-1 2024-09-14T03:41:11.631 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/ceph-2 on ubuntu@smithi083.front.sepia.ceph.com 2024-09-14T03:41:11.632 DEBUG:teuthology.orchestra.run.smithi083:> sync && sudo umount -f /var/lib/ceph/osd/ceph-2 2024-09-14T03:41:11.693 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/ceph-3 on ubuntu@smithi083.front.sepia.ceph.com 2024-09-14T03:41:11.693 DEBUG:teuthology.orchestra.run.smithi083:> sync && sudo umount -f /var/lib/ceph/osd/ceph-3 2024-09-14T03:41:11.756 INFO:tasks.ceph:Archiving mon data... 2024-09-14T03:41:11.758 DEBUG:teuthology.misc:Transferring archived files from smithi083:/var/lib/ceph/mon/ceph-a to /home/teuthworker/archive/teuthology-2024-09-14_03:08:15-upgrade-main-distro-default-smithi/7904542/data/mon.a.tgz 2024-09-14T03:41:11.758 DEBUG:teuthology.orchestra.run.smithi083:> mktemp 2024-09-14T03:41:11.773 INFO:teuthology.orchestra.run.smithi083.stdout:/tmp/tmp.LZUjbWqUVq 2024-09-14T03:41:11.773 DEBUG:teuthology.orchestra.run.smithi083:> sudo tar cz -f - -C /var/lib/ceph/mon/ceph-a -- . > /tmp/tmp.LZUjbWqUVq 2024-09-14T03:41:11.891 DEBUG:teuthology.orchestra.run.smithi083:> sudo chmod 0666 /tmp/tmp.LZUjbWqUVq 2024-09-14T03:41:11.974 DEBUG:teuthology.orchestra.remote:smithi083:/tmp/tmp.LZUjbWqUVq is 243KB 2024-09-14T03:41:12.069 DEBUG:teuthology.orchestra.run.smithi083:> rm -fr /tmp/tmp.LZUjbWqUVq 2024-09-14T03:41:12.084 DEBUG:teuthology.misc:Transferring archived files from smithi083:/var/lib/ceph/mon/ceph-b to /home/teuthworker/archive/teuthology-2024-09-14_03:08:15-upgrade-main-distro-default-smithi/7904542/data/mon.b.tgz 2024-09-14T03:41:12.084 DEBUG:teuthology.orchestra.run.smithi083:> mktemp 2024-09-14T03:41:12.139 INFO:teuthology.orchestra.run.smithi083.stdout:/tmp/tmp.OK0jyUS2KV 2024-09-14T03:41:12.140 DEBUG:teuthology.orchestra.run.smithi083:> sudo tar cz -f - -C /var/lib/ceph/mon/ceph-b -- . > /tmp/tmp.OK0jyUS2KV 2024-09-14T03:41:12.257 DEBUG:teuthology.orchestra.run.smithi083:> sudo chmod 0666 /tmp/tmp.OK0jyUS2KV 2024-09-14T03:41:12.341 DEBUG:teuthology.orchestra.remote:smithi083:/tmp/tmp.OK0jyUS2KV is 290KB 2024-09-14T03:41:12.436 DEBUG:teuthology.orchestra.run.smithi083:> rm -fr /tmp/tmp.OK0jyUS2KV 2024-09-14T03:41:12.451 DEBUG:teuthology.misc:Transferring archived files from smithi083:/var/lib/ceph/mon/ceph-c to /home/teuthworker/archive/teuthology-2024-09-14_03:08:15-upgrade-main-distro-default-smithi/7904542/data/mon.c.tgz 2024-09-14T03:41:12.451 DEBUG:teuthology.orchestra.run.smithi083:> mktemp 2024-09-14T03:41:12.505 INFO:teuthology.orchestra.run.smithi083.stdout:/tmp/tmp.rtvtIYKLTw 2024-09-14T03:41:12.505 DEBUG:teuthology.orchestra.run.smithi083:> sudo tar cz -f - -C /var/lib/ceph/mon/ceph-c -- . > /tmp/tmp.rtvtIYKLTw 2024-09-14T03:41:12.623 DEBUG:teuthology.orchestra.run.smithi083:> sudo chmod 0666 /tmp/tmp.rtvtIYKLTw 2024-09-14T03:41:12.705 DEBUG:teuthology.orchestra.remote:smithi083:/tmp/tmp.rtvtIYKLTw is 290KB 2024-09-14T03:41:12.799 DEBUG:teuthology.orchestra.run.smithi083:> rm -fr /tmp/tmp.rtvtIYKLTw 2024-09-14T03:41:12.814 INFO:tasks.ceph:Cleaning ceph cluster... 2024-09-14T03:41:12.815 DEBUG:teuthology.orchestra.run.smithi083:> sudo rm -rf -- /etc/ceph/ceph.conf /etc/ceph/ceph.keyring /home/ubuntu/cephtest/ceph.data /home/ubuntu/cephtest/ceph.monmap /home/ubuntu/cephtest/../*.pid 2024-09-14T03:41:12.934 INFO:teuthology.util.scanner:summary_data or yaml_file is empty! 2024-09-14T03:41:12.934 INFO:tasks.ceph:Archiving crash dumps... 2024-09-14T03:41:12.936 DEBUG:teuthology.misc:Transferring archived files from smithi083:/var/lib/ceph/crash to /home/teuthworker/archive/teuthology-2024-09-14_03:08:15-upgrade-main-distro-default-smithi/7904542/remote/smithi083/crash 2024-09-14T03:41:12.937 DEBUG:teuthology.orchestra.run.smithi083:> sudo tar c -f - -C /var/lib/ceph/crash -- . 2024-09-14T03:41:12.998 INFO:tasks.ceph:Compressing logs... 2024-09-14T03:41:12.998 DEBUG:teuthology.orchestra.run.smithi083:> time sudo find /var/log/ceph -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-09-14T03:41:13.063 INFO:teuthology.orchestra.run.smithi083.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-osd.0.log 2024-09-14T03:41:13.063 INFO:teuthology.orchestra.run.smithi083.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.39838.log 2024-09-14T03:41:13.063 INFO:teuthology.orchestra.run.smithi083.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.41214.log 2024-09-14T03:41:13.063 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-osd.0.log: /var/log/ceph/ceph-client.admin.39838.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.39670.log 2024-09-14T03:41:13.103 INFO:teuthology.orchestra.run.smithi083.stderr: 87.2% -- replaced with /var/log/ceph/ceph-client.admin.39838.log.gz 2024-09-14T03:41:13.103 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.41214.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.47376.log 2024-09-14T03:41:13.103 INFO:teuthology.orchestra.run.smithi083.stderr: 86.9% -- replaced with /var/log/ceph/ceph-client.admin.41214.log.gz 2024-09-14T03:41:13.103 INFO:teuthology.orchestra.run.smithi083.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.42325.log 2024-09-14T03:41:13.103 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.39670.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.46638.log 2024-09-14T03:41:13.103 INFO:teuthology.orchestra.run.smithi083.stderr: 86.8% -- replaced with /var/log/ceph/ceph-client.admin.39670.log.gz 2024-09-14T03:41:13.103 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.47376.log: /var/log/ceph/ceph-client.admin.42325.log: 87.2% -- replaced with /var/log/ceph/ceph-client.admin.47376.log.gz 2024-09-14T03:41:13.103 INFO:teuthology.orchestra.run.smithi083.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.32678.log 2024-09-14T03:41:13.103 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.46638.log: 87.1% -- replaced with /var/log/ceph/ceph-client.admin.42325.log.gz 2024-09-14T03:41:13.103 INFO:teuthology.orchestra.run.smithi083.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.39033.log 2024-09-14T03:41:13.104 INFO:teuthology.orchestra.run.smithi083.stderr: 86.8% -- replaced with /var/log/ceph/ceph-client.admin.46638.log.gz 2024-09-14T03:41:13.104 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.32678.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.41257.log 2024-09-14T03:41:13.104 INFO:teuthology.orchestra.run.smithi083.stderr: 79.1% -- replaced with /var/log/ceph/ceph-client.admin.32678.log.gz 2024-09-14T03:41:13.104 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.39033.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.46117.log 2024-09-14T03:41:13.104 INFO:teuthology.orchestra.run.smithi083.stderr: 86.8% -- replaced with /var/log/ceph/ceph-client.admin.39033.log.gz/var/log/ceph/ceph-client.admin.41257.log: 2024-09-14T03:41:13.104 INFO:teuthology.orchestra.run.smithi083.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.46077.log 2024-09-14T03:41:13.104 INFO:teuthology.orchestra.run.smithi083.stderr: 86.6% -- replaced with /var/log/ceph/ceph-client.admin.41257.log.gz 2024-09-14T03:41:13.104 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.46117.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.41358.log 2024-09-14T03:41:13.104 INFO:teuthology.orchestra.run.smithi083.stderr: 87.2% -- replaced with /var/log/ceph/ceph-client.admin.46117.log.gz 2024-09-14T03:41:13.104 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.46077.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.41894.log 2024-09-14T03:41:13.104 INFO:teuthology.orchestra.run.smithi083.stderr: 87.5% -- replaced with /var/log/ceph/ceph-client.admin.46077.log.gz 2024-09-14T03:41:13.104 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.41358.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.42093.log 2024-09-14T03:41:13.104 INFO:teuthology.orchestra.run.smithi083.stderr: 85.4% -- replaced with /var/log/ceph/ceph-client.admin.41358.log.gz 2024-09-14T03:41:13.104 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.41894.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.40170.log 2024-09-14T03:41:13.105 INFO:teuthology.orchestra.run.smithi083.stderr: 86.8% -- replaced with /var/log/ceph/ceph-client.admin.41894.log.gz 2024-09-14T03:41:13.105 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.42093.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.40006.log 2024-09-14T03:41:13.105 INFO:teuthology.orchestra.run.smithi083.stderr: 87.1%/var/log/ceph/ceph-client.admin.40170.log: -- replaced with /var/log/ceph/ceph-client.admin.42093.log.gz 2024-09-14T03:41:13.105 INFO:teuthology.orchestra.run.smithi083.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.47836.log 2024-09-14T03:41:13.105 INFO:teuthology.orchestra.run.smithi083.stderr: 87.3% -- replaced with /var/log/ceph/ceph-client.admin.40170.log.gz 2024-09-14T03:41:13.105 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.40006.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.39728.log 2024-09-14T03:41:13.105 INFO:teuthology.orchestra.run.smithi083.stderr: 86.1% -- replaced with /var/log/ceph/ceph-client.admin.40006.log.gz 2024-09-14T03:41:13.105 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.47836.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.39114.log 2024-09-14T03:41:13.105 INFO:teuthology.orchestra.run.smithi083.stderr: 86.9% -- replaced with /var/log/ceph/ceph-client.admin.47836.log.gz 2024-09-14T03:41:13.105 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.39728.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.46511.log 2024-09-14T03:41:13.105 INFO:teuthology.orchestra.run.smithi083.stderr: 87.0% -- replaced with /var/log/ceph/ceph-client.admin.39728.log.gz 2024-09-14T03:41:13.105 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.39114.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.46261.log 2024-09-14T03:41:13.105 INFO:teuthology.orchestra.run.smithi083.stderr: 85.4% -- replaced with /var/log/ceph/ceph-client.admin.39114.log.gz 2024-09-14T03:41:13.105 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.46511.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.44389.log 2024-09-14T03:41:13.106 INFO:teuthology.orchestra.run.smithi083.stderr: 85.6% -- replaced with /var/log/ceph/ceph-client.admin.46511.log.gz/var/log/ceph/ceph-client.admin.46261.log: 2024-09-14T03:41:13.106 INFO:teuthology.orchestra.run.smithi083.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-osd.1.log 2024-09-14T03:41:13.106 INFO:teuthology.orchestra.run.smithi083.stderr: 86.6% -- replaced with /var/log/ceph/ceph-client.admin.46261.log.gz 2024-09-14T03:41:13.106 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.44389.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.39392.log 2024-09-14T03:41:13.106 INFO:teuthology.orchestra.run.smithi083.stderr: 86.7% -- replaced with /var/log/ceph/ceph-client.admin.44389.log.gz 2024-09-14T03:41:13.106 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-osd.1.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.46410.log 2024-09-14T03:41:13.106 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.39392.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.41933.log 2024-09-14T03:41:13.106 INFO:teuthology.orchestra.run.smithi083.stderr: 85.8% -- replaced with /var/log/ceph/ceph-client.admin.39392.log.gz 2024-09-14T03:41:13.106 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.46410.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.42150.log 2024-09-14T03:41:13.106 INFO:teuthology.orchestra.run.smithi083.stderr: 87.0% -- replaced with /var/log/ceph/ceph-client.admin.46410.log.gz 2024-09-14T03:41:13.106 INFO:teuthology.orchestra.run.smithi083.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.32744.log 2024-09-14T03:41:13.106 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.41933.log: 86.9% -- replaced with /var/log/ceph/ceph-client.admin.41933.log.gz 2024-09-14T03:41:13.106 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.42150.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.32708.log 2024-09-14T03:41:13.106 INFO:teuthology.orchestra.run.smithi083.stderr: 86.4% -- replaced with /var/log/ceph/ceph-client.admin.42150.log.gz 2024-09-14T03:41:13.107 INFO:teuthology.orchestra.run.smithi083.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.39207.log 2024-09-14T03:41:13.107 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.32744.log: 77.0% -- replaced with /var/log/ceph/ceph-client.admin.32744.log.gz 2024-09-14T03:41:13.107 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.32708.log: gzip -5 --verbose -- /var/log/ceph/ceph.tmp-client.admin.26583.log 2024-09-14T03:41:13.107 INFO:teuthology.orchestra.run.smithi083.stderr: 77.5% -- replaced with /var/log/ceph/ceph-client.admin.32708.log.gz 2024-09-14T03:41:13.107 INFO:teuthology.orchestra.run.smithi083.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.47208.log 2024-09-14T03:41:13.107 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.39207.log: /var/log/ceph/ceph.tmp-client.admin.26583.log: gzip 0.0% -5 -- replaced with /var/log/ceph/ceph.tmp-client.admin.26583.log.gz --verbose 2024-09-14T03:41:13.107 INFO:teuthology.orchestra.run.smithi083.stderr: -- /var/log/ceph/ceph-client.admin.33114.log 2024-09-14T03:41:13.107 INFO:teuthology.orchestra.run.smithi083.stderr: 87.2% -- replaced with /var/log/ceph/ceph-client.admin.39207.log.gz 2024-09-14T03:41:13.107 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.47208.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.33098.log 2024-09-14T03:41:13.107 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.33114.log: 86.6% -- replaced with /var/log/ceph/ceph-client.admin.47208.log.gz 2024-09-14T03:41:13.107 INFO:teuthology.orchestra.run.smithi083.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.39089.log 2024-09-14T03:41:13.107 INFO:teuthology.orchestra.run.smithi083.stderr: 86.6% -- replaced with /var/log/ceph/ceph-client.admin.33114.log.gz 2024-09-14T03:41:13.107 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.33098.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.46284.log 2024-09-14T03:41:13.108 INFO:teuthology.orchestra.run.smithi083.stderr: 86.8% -- replaced with /var/log/ceph/ceph-client.admin.33098.log.gz 2024-09-14T03:41:13.108 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.39089.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.46510.log 2024-09-14T03:41:13.108 INFO:teuthology.orchestra.run.smithi083.stderr: 87.0% -- replaced with /var/log/ceph/ceph-client.admin.39089.log.gz 2024-09-14T03:41:13.108 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.46284.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.41131.log 2024-09-14T03:41:13.108 INFO:teuthology.orchestra.run.smithi083.stderr: 86.6% -- replaced with /var/log/ceph/ceph-client.admin.46284.log.gz 2024-09-14T03:41:13.108 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.46510.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.47670.log 2024-09-14T03:41:13.108 INFO:teuthology.orchestra.run.smithi083.stderr: 85.7% -- replaced with /var/log/ceph/ceph-client.admin.46510.log.gz 2024-09-14T03:41:13.108 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.41131.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.42519.log 2024-09-14T03:41:13.108 INFO:teuthology.orchestra.run.smithi083.stderr: 87.2% -- replaced with /var/log/ceph/ceph-client.admin.41131.log.gz 2024-09-14T03:41:13.108 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.47670.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.42613.log 2024-09-14T03:41:13.108 INFO:teuthology.orchestra.run.smithi083.stderr: 86.0% -- replaced with /var/log/ceph/ceph-client.admin.47670.log.gz 2024-09-14T03:41:13.108 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.42519.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.41934.log 2024-09-14T03:41:13.108 INFO:teuthology.orchestra.run.smithi083.stderr: 86.6% -- replaced with /var/log/ceph/ceph-client.admin.42519.log.gz 2024-09-14T03:41:13.108 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.42613.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.42283.log 2024-09-14T03:41:13.109 INFO:teuthology.orchestra.run.smithi083.stderr: 86.4% -- replaced with /var/log/ceph/ceph-client.admin.42613.log.gz 2024-09-14T03:41:13.109 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.41934.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.32502.log 2024-09-14T03:41:13.109 INFO:teuthology.orchestra.run.smithi083.stderr: 86.8% -- replaced with /var/log/ceph/ceph-client.admin.41934.log.gz 2024-09-14T03:41:13.109 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.42283.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.46512.log 2024-09-14T03:41:13.109 INFO:teuthology.orchestra.run.smithi083.stderr: 86.4% -- replaced with /var/log/ceph/ceph-client.admin.42283.log.gz 2024-09-14T03:41:13.109 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.32502.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.40130.log 2024-09-14T03:41:13.109 INFO:teuthology.orchestra.run.smithi083.stderr: 76.1% -- replaced with /var/log/ceph/ceph-client.admin.32502.log.gz 2024-09-14T03:41:13.109 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.46512.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.39729.log 2024-09-14T03:41:13.109 INFO:teuthology.orchestra.run.smithi083.stderr: 85.8% -- replaced with /var/log/ceph/ceph-client.admin.46512.log.gz 2024-09-14T03:41:13.109 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.40130.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.39907.log 2024-09-14T03:41:13.109 INFO:teuthology.orchestra.run.smithi083.stderr: 87.0% -- replaced with /var/log/ceph/ceph-client.admin.40130.log.gz 2024-09-14T03:41:13.109 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.39729.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.47586.log 2024-09-14T03:41:13.109 INFO:teuthology.orchestra.run.smithi083.stderr: 86.7% -- replaced with /var/log/ceph/ceph-client.admin.39729.log.gz 2024-09-14T03:41:13.109 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.39907.log: gzip -5 --verbose -- /var/log/ceph/ceph.audit.log 2024-09-14T03:41:13.110 INFO:teuthology.orchestra.run.smithi083.stderr: 86.7% -- replaced with /var/log/ceph/ceph-client.admin.39907.log.gz 2024-09-14T03:41:13.110 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.47586.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.46157.log 2024-09-14T03:41:13.110 INFO:teuthology.orchestra.run.smithi083.stderr: 85.4% -- replaced with /var/log/ceph/ceph-client.admin.47586.log.gz 2024-09-14T03:41:13.110 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.47502.log 2024-09-14T03:41:13.110 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.46157.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.46640.log 2024-09-14T03:41:13.110 INFO:teuthology.orchestra.run.smithi083.stderr: 86.8% -- replaced with /var/log/ceph/ceph-client.admin.46157.log.gz 2024-09-14T03:41:13.110 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.47502.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.33598.log 2024-09-14T03:41:13.110 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.46640.log: 85.7% -- replaced with /var/log/ceph/ceph-client.admin.47502.log.gz 2024-09-14T03:41:13.110 INFO:teuthology.orchestra.run.smithi083.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.46368.log 2024-09-14T03:41:13.110 INFO:teuthology.orchestra.run.smithi083.stderr: 86.8% -- replaced with /var/log/ceph/ceph-client.admin.46640.log.gz 2024-09-14T03:41:13.110 INFO:teuthology.orchestra.run.smithi083.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.47166.log 2024-09-14T03:41:13.110 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.33598.log: 95.0% -- replaced with /var/log/ceph/ceph.audit.log.gz 2024-09-14T03:41:13.110 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.46368.log: 87.3% -- replaced with /var/log/ceph/ceph-client.admin.33598.log.gz 2024-09-14T03:41:13.110 INFO:teuthology.orchestra.run.smithi083.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.41359.log 2024-09-14T03:41:13.111 INFO:teuthology.orchestra.run.smithi083.stderr: 86.9% -- replaced with /var/log/ceph/ceph-client.admin.46368.log.gz 2024-09-14T03:41:13.111 INFO:teuthology.orchestra.run.smithi083.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.41726.log 2024-09-14T03:41:13.111 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.47166.log: /var/log/ceph/ceph-client.admin.41359.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.32956.log 2024-09-14T03:41:13.111 INFO:teuthology.orchestra.run.smithi083.stderr: 86.2% -- replaced with /var/log/ceph/ceph-client.admin.47166.log.gz 2024-09-14T03:41:13.111 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.41726.log: 85.5% -- replaced with /var/log/ceph/ceph-client.admin.41359.log.gz 2024-09-14T03:41:13.111 INFO:teuthology.orchestra.run.smithi083.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.47292.log 2024-09-14T03:41:13.111 INFO:teuthology.orchestra.run.smithi083.stderr: 86.8% -- replaced with /var/log/ceph/ceph-client.admin.41726.log.gz 2024-09-14T03:41:13.111 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.32956.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.46808.log 2024-09-14T03:41:13.111 INFO:teuthology.orchestra.run.smithi083.stderr: 85.9% -- replaced with /var/log/ceph/ceph-client.admin.32956.log.gz 2024-09-14T03:41:13.111 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.47292.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.41747.log 2024-09-14T03:41:13.111 INFO:teuthology.orchestra.run.smithi083.stderr: 86.5% -- replaced with /var/log/ceph/ceph-client.admin.47292.log.gz 2024-09-14T03:41:13.111 INFO:teuthology.orchestra.run.smithi083.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.33558.log 2024-09-14T03:41:13.111 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.46808.log: /var/log/ceph/ceph-client.admin.41747.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.42073.log 2024-09-14T03:41:13.111 INFO:teuthology.orchestra.run.smithi083.stderr: 86.6% -- replaced with /var/log/ceph/ceph-client.admin.46808.log.gz 2024-09-14T03:41:13.112 INFO:teuthology.orchestra.run.smithi083.stderr: 87.2% -- replaced with /var/log/ceph/ceph-client.admin.41747.log.gz 2024-09-14T03:41:13.112 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.33558.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.39391.log 2024-09-14T03:41:13.112 INFO:teuthology.orchestra.run.smithi083.stderr: 86.8% -- replaced with /var/log/ceph/ceph-client.admin.33558.log.gz 2024-09-14T03:41:13.112 INFO:teuthology.orchestra.run.smithi083.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.41510.log 2024-09-14T03:41:13.112 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.42073.log: 87.0% -- replaced with /var/log/ceph/ceph-client.admin.42073.log.gz 2024-09-14T03:41:13.112 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.39391.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.39730.log 2024-09-14T03:41:13.112 INFO:teuthology.orchestra.run.smithi083.stderr: 85.3% -- replaced with /var/log/ceph/ceph-client.admin.39391.log.gz 2024-09-14T03:41:13.112 INFO:teuthology.orchestra.run.smithi083.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.39291.log 2024-09-14T03:41:13.112 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.41510.log: 86.9% -- replaced with /var/log/ceph/ceph-client.admin.41510.log.gz 2024-09-14T03:41:13.112 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.39730.log: gzip -5 --verbose -- /var/log/ceph/ceph-mon.c.log 2024-09-14T03:41:13.112 INFO:teuthology.orchestra.run.smithi083.stderr: 86.9% -- replaced with /var/log/ceph/ceph-client.admin.39730.log.gz 2024-09-14T03:41:13.112 INFO:teuthology.orchestra.run.smithi083.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.39558.log 2024-09-14T03:41:13.112 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.39291.log: 86.8% -- replaced with /var/log/ceph/ceph-client.admin.39291.log.gz 2024-09-14T03:41:13.113 INFO:teuthology.orchestra.run.smithi083.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.39390.log 2024-09-14T03:41:13.113 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-mon.c.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.32772.log 2024-09-14T03:41:13.113 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.39558.log: /var/log/ceph/ceph-client.admin.39390.log: 86.7% -- replaced with /var/log/ceph/ceph-client.admin.39558.log.gz 2024-09-14T03:41:13.113 INFO:teuthology.orchestra.run.smithi083.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.32965.log 2024-09-14T03:41:13.113 INFO:teuthology.orchestra.run.smithi083.stderr: 85.8% -- replaced with /var/log/ceph/ceph-client.admin.39390.log.gz 2024-09-14T03:41:13.113 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.32772.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.33366.log 2024-09-14T03:41:13.113 INFO:teuthology.orchestra.run.smithi083.stderr: 0.0% -- replaced with /var/log/ceph/ceph-client.admin.32772.log.gz 2024-09-14T03:41:13.113 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.32965.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.33013.log 2024-09-14T03:41:13.113 INFO:teuthology.orchestra.run.smithi083.stderr: 85.9%/var/log/ceph/ceph-client.admin.33366.log: -- replaced with /var/log/ceph/ceph-client.admin.32965.log.gz 2024-09-14T03:41:13.113 INFO:teuthology.orchestra.run.smithi083.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.41356.log 2024-09-14T03:41:13.113 INFO:teuthology.orchestra.run.smithi083.stderr: 86.6% -- replaced with /var/log/ceph/ceph-client.admin.33366.log.gz 2024-09-14T03:41:13.113 INFO:teuthology.orchestra.run.smithi083.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.46975.log 2024-09-14T03:41:13.113 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.33013.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.32650.log 2024-09-14T03:41:13.113 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.41356.log: 86.7% -- replaced with /var/log/ceph/ceph-client.admin.33013.log.gz 2024-09-14T03:41:13.114 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.46975.log: gzip 85.5% -5 --verbose -- replaced with /var/log/ceph/ceph-client.admin.41356.log.gz -- 2024-09-14T03:41:13.114 INFO:teuthology.orchestra.run.smithi083.stderr: /var/log/ceph/ceph-client.admin.39560.log 2024-09-14T03:41:13.114 INFO:teuthology.orchestra.run.smithi083.stderr: 86.7% -- replaced with /var/log/ceph/ceph-client.admin.46975.log.gz 2024-09-14T03:41:13.114 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.32650.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.39561.log 2024-09-14T03:41:13.114 INFO:teuthology.orchestra.run.smithi083.stderr: 75.0% -- replaced with /var/log/ceph/ceph-client.admin.32650.log.gz 2024-09-14T03:41:13.114 INFO:teuthology.orchestra.run.smithi083.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.33478.log 2024-09-14T03:41:13.114 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.39560.log: /var/log/ceph/ceph-client.admin.39561.log: 86.8% -- replaced with /var/log/ceph/ceph-client.admin.39560.log.gz 2024-09-14T03:41:13.114 INFO:teuthology.orchestra.run.smithi083.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.47124.log 2024-09-14T03:41:13.114 INFO:teuthology.orchestra.run.smithi083.stderr: 86.6% -- replaced with /var/log/ceph/ceph-client.admin.39561.log.gz 2024-09-14T03:41:13.114 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.33478.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.46788.log 2024-09-14T03:41:13.114 INFO:teuthology.orchestra.run.smithi083.stderr: 86.9% -- replaced with /var/log/ceph/ceph-client.admin.33478.log.gz 2024-09-14T03:41:13.114 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.47124.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.32884.log 2024-09-14T03:41:13.114 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.46788.log: 85.8% -- replaced with /var/log/ceph/ceph-client.admin.47124.log.gz 2024-09-14T03:41:13.114 INFO:teuthology.orchestra.run.smithi083.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.42151.log 2024-09-14T03:41:13.115 INFO:teuthology.orchestra.run.smithi083.stderr: 87.3% -- replaced with /var/log/ceph/ceph-client.admin.46788.log.gz 2024-09-14T03:41:13.115 INFO:teuthology.orchestra.run.smithi083.stderr:gzip -5 --verbose/var/log/ceph/ceph-client.admin.32884.log: -- /var/log/ceph/ceph-client.admin.32478.log 2024-09-14T03:41:13.115 INFO:teuthology.orchestra.run.smithi083.stderr: 78.8% -- replaced with /var/log/ceph/ceph-client.admin.32884.log.gz 2024-09-14T03:41:13.115 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.42151.log: gzip -5 --verbose -- /var/log/ceph/ceph-mon.a.log 2024-09-14T03:41:13.115 INFO:teuthology.orchestra.run.smithi083.stderr: 86.9% -- replaced with /var/log/ceph/ceph-client.admin.42151.log.gz 2024-09-14T03:41:13.115 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.32478.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.46238.log 2024-09-14T03:41:13.115 INFO:teuthology.orchestra.run.smithi083.stderr: 87.1% -- replaced with /var/log/ceph/ceph-client.admin.32478.log.gz 2024-09-14T03:41:13.115 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-mon.a.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.33518.log 2024-09-14T03:41:13.115 INFO:teuthology.orchestra.run.smithi083.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.37960.log 2024-09-14T03:41:13.115 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.46238.log: /var/log/ceph/ceph-client.admin.33518.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.46037.log 2024-09-14T03:41:13.115 INFO:teuthology.orchestra.run.smithi083.stderr: 87.2% -- replaced with /var/log/ceph/ceph-client.admin.46238.log.gz 2024-09-14T03:41:13.115 INFO:teuthology.orchestra.run.smithi083.stderr: 86.9% -- replaced with /var/log/ceph/ceph-client.admin.33518.log.gz 2024-09-14T03:41:13.115 INFO:teuthology.orchestra.run.smithi083.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.39249.log 2024-09-14T03:41:13.116 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.37960.log: /var/log/ceph/ceph-client.admin.46037.log: 86.7%gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.46680.log 2024-09-14T03:41:13.116 INFO:teuthology.orchestra.run.smithi083.stderr: -- replaced with /var/log/ceph/ceph-client.admin.37960.log.gz 2024-09-14T03:41:13.116 INFO:teuthology.orchestra.run.smithi083.stderr: 87.2% -- replaced with /var/log/ceph/ceph-client.admin.46037.log.gz 2024-09-14T03:41:13.116 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.39249.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.33123.log 2024-09-14T03:41:13.116 INFO:teuthology.orchestra.run.smithi083.stderr: 86.9% -- replaced with /var/log/ceph/ceph-client.admin.39249.log.gz 2024-09-14T03:41:13.116 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.46680.log: gzip -5 --verbose -- /var/log/ceph/ceph-osd.2.log 2024-09-14T03:41:13.116 INFO:teuthology.orchestra.run.smithi083.stderr:gzip 87.4% -- replaced with /var/log/ceph/ceph-client.admin.46680.log.gz 2024-09-14T03:41:13.116 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.33123.log: -5 --verbose -- /var/log/ceph/ceph-client.admin.41974.log 2024-09-14T03:41:13.116 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-osd.2.log: 86.6% -- replaced with /var/log/ceph/ceph-client.admin.33123.log.gz 2024-09-14T03:41:13.116 INFO:teuthology.orchestra.run.smithi083.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.46976.log 2024-09-14T03:41:13.116 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.41974.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.40048.log 2024-09-14T03:41:13.116 INFO:teuthology.orchestra.run.smithi083.stderr: 87.2% -- replaced with /var/log/ceph/ceph-client.admin.41974.log.gz/var/log/ceph/ceph-client.admin.46976.log: 2024-09-14T03:41:13.117 INFO:teuthology.orchestra.run.smithi083.stderr:gzip -5 --verbose -- 87.2% -- replaced with /var/log/ceph/ceph-client.admin.46976.log.gz 2024-09-14T03:41:13.117 INFO:teuthology.orchestra.run.smithi083.stderr: /var/log/ceph/ceph-client.admin.47796.log 2024-09-14T03:41:13.117 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.40048.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.41468.log 86.5% -- replaced with /var/log/ceph/ceph-client.admin.40048.log.gz 2024-09-14T03:41:13.117 INFO:teuthology.orchestra.run.smithi083.stderr: 2024-09-14T03:41:13.117 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.47796.log: gzip -5 --verbose -- /var/log/ceph/ceph-mgr.y.log 2024-09-14T03:41:13.117 INFO:teuthology.orchestra.run.smithi083.stderr: 86.0% -- replaced with /var/log/ceph/ceph-client.admin.47796.log.gz 2024-09-14T03:41:13.117 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.41468.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.47754.log 2024-09-14T03:41:13.117 INFO:teuthology.orchestra.run.smithi083.stderr: 86.9% -- replaced with /var/log/ceph/ceph-client.admin.41468.log.gz 2024-09-14T03:41:13.117 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-mgr.y.log: gzip -5 --verbose -- /var/log/ceph/ceph.log 2024-09-14T03:41:13.117 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.47754.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.39165.log 2024-09-14T03:41:13.117 INFO:teuthology.orchestra.run.smithi083.stderr: 85.9% -- replaced with /var/log/ceph/ceph-client.admin.47754.log.gz 2024-09-14T03:41:13.117 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.46849.log 2024-09-14T03:41:13.117 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.39165.log: 87.1% -- replaced with /var/log/ceph/ceph-client.admin.39165.log.gz 2024-09-14T03:41:13.117 INFO:teuthology.orchestra.run.smithi083.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.46326.log 2024-09-14T03:41:13.118 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.46849.log: 93.8% -- replaced with /var/log/ceph/ceph.log.gz 2024-09-14T03:41:13.118 INFO:teuthology.orchestra.run.smithi083.stderr: 86.9% -- replaced with /var/log/ceph/ceph-client.admin.46849.log.gz 2024-09-14T03:41:13.118 INFO:teuthology.orchestra.run.smithi083.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-mgr.x.log 2024-09-14T03:41:13.118 INFO:teuthology.orchestra.run.smithi083.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.40090.log 2024-09-14T03:41:13.118 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.46326.log: /var/log/ceph/ceph-mgr.x.log: 86.6% -- replaced with /var/log/ceph/ceph-client.admin.46326.log.gz 2024-09-14T03:41:13.118 INFO:teuthology.orchestra.run.smithi083.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.47250.log 2024-09-14T03:41:13.118 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.40090.log: 86.6% -- replaced with /var/log/ceph/ceph-client.admin.40090.log.gz 2024-09-14T03:41:13.118 INFO:teuthology.orchestra.run.smithi083.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.47460.log 2024-09-14T03:41:13.118 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.47250.log: 86.7% -- replaced with /var/log/ceph/ceph-client.admin.47250.log.gz 2024-09-14T03:41:13.118 INFO:teuthology.orchestra.run.smithi083.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.46850.log 2024-09-14T03:41:13.118 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.47460.log: 85.8% -- replaced with /var/log/ceph/ceph-client.admin.47460.log.gz 2024-09-14T03:41:13.118 INFO:teuthology.orchestra.run.smithi083.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.46996.log 2024-09-14T03:41:13.118 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.46850.log: 86.6% -- replaced with /var/log/ceph/ceph-client.admin.46850.log.gz 2024-09-14T03:41:13.118 INFO:teuthology.orchestra.run.smithi083.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.47544.log 2024-09-14T03:41:13.119 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.46996.log: 86.6% -- replaced with /var/log/ceph/ceph-client.admin.46996.log.gz 2024-09-14T03:41:13.142 INFO:teuthology.orchestra.run.smithi083.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.42241.log 2024-09-14T03:41:13.143 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.47544.log: 86.2% -- replaced with /var/log/ceph/ceph-client.admin.47544.log.gz 2024-09-14T03:41:13.143 INFO:teuthology.orchestra.run.smithi083.stderr: 91.8% -- replaced with /var/log/ceph/ceph-mgr.x.log.gz 2024-09-14T03:41:13.143 INFO:teuthology.orchestra.run.smithi083.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.41509.log 2024-09-14T03:41:13.143 INFO:teuthology.orchestra.run.smithi083.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.39393.log 2024-09-14T03:41:13.143 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.41509.log: /var/log/ceph/ceph-client.admin.42241.log: 86.5% -- replaced with /var/log/ceph/ceph-client.admin.42241.log.gz 2024-09-14T03:41:13.143 INFO:teuthology.orchestra.run.smithi083.stderr: 87.1% -- replaced with /var/log/ceph/ceph-client.admin.41509.log.gz 2024-09-14T03:41:13.143 INFO:teuthology.orchestra.run.smithi083.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.41555.log 2024-09-14T03:41:13.143 INFO:teuthology.orchestra.run.smithi083.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-mon.b.log 2024-09-14T03:41:13.143 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.39393.log: /var/log/ceph/ceph-client.admin.41555.log: 85.8% -- replaced with /var/log/ceph/ceph-client.admin.39393.log.gz 2024-09-14T03:41:13.143 INFO:teuthology.orchestra.run.smithi083.stderr: 86.6% -- replaced with /var/log/ceph/ceph-client.admin.41555.log.gz 2024-09-14T03:41:13.143 INFO:teuthology.orchestra.run.smithi083.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.41357.log 2024-09-14T03:41:13.143 INFO:teuthology.orchestra.run.smithi083.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.33111.log 2024-09-14T03:41:13.144 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.41357.log: /var/log/ceph/ceph-mon.b.log: 85.9% -- replaced with /var/log/ceph/ceph-client.admin.41357.log.gz 2024-09-14T03:41:13.144 INFO:teuthology.orchestra.run.smithi083.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.33799.log 2024-09-14T03:41:13.144 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.33111.log: 86.4% -- replaced with /var/log/ceph/ceph-client.admin.33111.log.gz 2024-09-14T03:41:13.144 INFO:teuthology.orchestra.run.smithi083.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.46197.log 2024-09-14T03:41:13.144 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.33799.log: 86.7% -- replaced with /var/log/ceph/ceph-client.admin.33799.log.gz 2024-09-14T03:41:13.144 INFO:teuthology.orchestra.run.smithi083.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.39908.log 2024-09-14T03:41:13.144 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.46197.log: 86.7% -- replaced with /var/log/ceph/ceph-client.admin.46197.log.gz 2024-09-14T03:41:13.144 INFO:teuthology.orchestra.run.smithi083.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.41805.log 2024-09-14T03:41:13.144 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.39908.log: 86.3% -- replaced with /var/log/ceph/ceph-client.admin.39908.log.gz 2024-09-14T03:41:13.144 INFO:teuthology.orchestra.run.smithi083.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.33127.log 2024-09-14T03:41:13.144 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.41805.log: 86.5% -- replaced with /var/log/ceph/ceph-client.admin.41805.log.gz 2024-09-14T03:41:13.144 INFO:teuthology.orchestra.run.smithi083.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.46679.log 2024-09-14T03:41:13.144 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.33127.log: 86.2% -- replaced with /var/log/ceph/ceph-client.admin.33127.log.gz 2024-09-14T03:41:13.144 INFO:teuthology.orchestra.run.smithi083.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.47418.log 2024-09-14T03:41:13.145 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.46679.log: 86.1% -- replaced with /var/log/ceph/ceph-client.admin.46679.log.gz 2024-09-14T03:41:13.145 INFO:teuthology.orchestra.run.smithi083.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.39858.log 2024-09-14T03:41:13.145 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.47418.log: 85.3% -- replaced with /var/log/ceph/ceph-client.admin.47418.log.gz 2024-09-14T03:41:13.145 INFO:teuthology.orchestra.run.smithi083.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.32800.log 2024-09-14T03:41:13.145 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.39858.log: 87.2% -- replaced with /var/log/ceph/ceph-client.admin.39858.log.gz 2024-09-14T03:41:13.145 INFO:teuthology.orchestra.run.smithi083.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.39559.log 2024-09-14T03:41:13.145 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.32800.log: 77.9% -- replaced with /var/log/ceph/ceph-client.admin.32800.log.gz 2024-09-14T03:41:13.145 INFO:teuthology.orchestra.run.smithi083.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.42367.log 2024-09-14T03:41:13.145 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.39559.log: 87.1% -- replaced with /var/log/ceph/ceph-client.admin.39559.log.gz 2024-09-14T03:41:13.145 INFO:teuthology.orchestra.run.smithi083.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.41806.log 2024-09-14T03:41:13.145 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.42367.log: 87.0% -- replaced with /var/log/ceph/ceph-client.admin.42367.log.gz 2024-09-14T03:41:13.145 INFO:teuthology.orchestra.run.smithi083.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.47334.log 2024-09-14T03:41:13.145 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.41806.log: 86.7% -- replaced with /var/log/ceph/ceph-client.admin.41806.log.gz 2024-09-14T03:41:13.145 INFO:teuthology.orchestra.run.smithi083.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.47712.log 2024-09-14T03:41:13.146 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.47334.log: 86.2% -- replaced with /var/log/ceph/ceph-client.admin.47334.log.gz 2024-09-14T03:41:13.146 INFO:teuthology.orchestra.run.smithi083.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.42440.log 2024-09-14T03:41:13.146 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.47712.log: 86.4% -- replaced with /var/log/ceph/ceph-client.admin.47712.log.gz 2024-09-14T03:41:13.146 INFO:teuthology.orchestra.run.smithi083.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-osd.3.log 2024-09-14T03:41:13.146 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.42440.log: 86.5% -- replaced with /var/log/ceph/ceph-client.admin.42440.log.gz 2024-09-14T03:41:13.146 INFO:teuthology.orchestra.run.smithi083.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.46509.log 2024-09-14T03:41:13.146 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-osd.3.log: 93.5% -- replaced with /var/log/ceph/ceph-osd.0.log.gz 2024-09-14T03:41:13.146 INFO:teuthology.orchestra.run.smithi083.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.33716.log 2024-09-14T03:41:13.146 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.46509.log: 85.6% -- replaced with /var/log/ceph/ceph-client.admin.46509.log.gz 2024-09-14T03:41:13.146 INFO:teuthology.orchestra.run.smithi083.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.42693.log 2024-09-14T03:41:13.146 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.33716.log: 87.1% -- replaced with /var/log/ceph/ceph-client.admin.33716.log.gz 2024-09-14T03:41:13.146 INFO:teuthology.orchestra.run.smithi083.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.33128.log 2024-09-14T03:41:13.146 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.42693.log: 86.9% -- replaced with /var/log/ceph/ceph-client.admin.42693.log.gz 2024-09-14T03:41:13.147 INFO:teuthology.orchestra.run.smithi083.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.47056.log 2024-09-14T03:41:13.147 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.33128.log: 87.1% -- replaced with /var/log/ceph/ceph-client.admin.33128.log.gz 2024-09-14T03:41:13.147 INFO:teuthology.orchestra.run.smithi083.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.45995.log 2024-09-14T03:41:13.147 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.47056.log: 86.8% -- replaced with /var/log/ceph/ceph-client.admin.47056.log.gz 2024-09-14T03:41:13.147 INFO:teuthology.orchestra.run.smithi083.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.47628.log 2024-09-14T03:41:13.147 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.45995.log: 87.3% -- replaced with /var/log/ceph/ceph-client.admin.45995.log.gz 2024-09-14T03:41:13.147 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/ceph-client.admin.47628.log: 86.7% -- replaced with /var/log/ceph/ceph-client.admin.47628.log.gz 2024-09-14T03:41:13.147 INFO:teuthology.orchestra.run.smithi083.stderr: 90.6% -- replaced with /var/log/ceph/ceph-mgr.y.log.gz 2024-09-14T03:41:13.147 INFO:teuthology.orchestra.run.smithi083.stderr: 93.1% -- replaced with /var/log/ceph/ceph-osd.2.log.gz 2024-09-14T03:41:13.159 INFO:teuthology.orchestra.run.smithi083.stderr: 93.7% -- replaced with /var/log/ceph/ceph-osd.1.log.gz 2024-09-14T03:41:13.167 INFO:teuthology.orchestra.run.smithi083.stderr: 91.7% -- replaced with /var/log/ceph/ceph-mon.c.log.gz 2024-09-14T03:41:13.184 INFO:teuthology.orchestra.run.smithi083.stderr: 93.3% -- replaced with /var/log/ceph/ceph-osd.3.log.gz 2024-09-14T03:41:13.186 INFO:teuthology.orchestra.run.smithi083.stderr: 91.6% -- replaced with /var/log/ceph/ceph-mon.b.log.gz 2024-09-14T03:41:13.212 INFO:teuthology.orchestra.run.smithi083.stderr: 90.9% -- replaced with /var/log/ceph/ceph-mon.a.log.gz 2024-09-14T03:41:13.214 INFO:teuthology.orchestra.run.smithi083.stderr: 2024-09-14T03:41:13.214 INFO:teuthology.orchestra.run.smithi083.stderr:real 0m0.161s 2024-09-14T03:41:13.214 INFO:teuthology.orchestra.run.smithi083.stderr:user 0m0.633s 2024-09-14T03:41:13.215 INFO:teuthology.orchestra.run.smithi083.stderr:sys 0m0.112s 2024-09-14T03:41:13.215 INFO:tasks.ceph:Archiving logs... 2024-09-14T03:41:13.215 DEBUG:teuthology.misc:Transferring archived files from smithi083:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-09-14_03:08:15-upgrade-main-distro-default-smithi/7904542/remote/smithi083/log 2024-09-14T03:41:13.216 DEBUG:teuthology.orchestra.run.smithi083:> sudo tar c -f - -C /var/log/ceph -- . 2024-09-14T03:41:13.464 DEBUG:teuthology.run_tasks:Unwinding manager install 2024-09-14T03:41:13.481 INFO:teuthology.task.install.util:Removing shipped files: /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer... 2024-09-14T03:41:13.481 DEBUG:teuthology.orchestra.run.smithi083:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2024-09-14T03:41:13.534 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-09-14T03:41:13.545 INFO:teuthology.task.clock:Checking final clock skew... 2024-09-14T03:41:13.545 DEBUG:teuthology.orchestra.run.smithi083:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-14T03:41:13.562 INFO:teuthology.orchestra.run.smithi083.stderr:bash: line 1: ntpq: command not found 2024-09-14T03:41:13.566 INFO:teuthology.orchestra.run.smithi083.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-14T03:41:13.567 INFO:teuthology.orchestra.run.smithi083.stdout:=============================================================================== 2024-09-14T03:41:13.567 INFO:teuthology.orchestra.run.smithi083.stdout:^* hv01.front.sepia.ceph.com 3 6 377 6 +153us[ +151us] +/- 40ms 2024-09-14T03:41:13.567 INFO:teuthology.orchestra.run.smithi083.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 4 -562us[ -562us] +/- 58ms 2024-09-14T03:41:13.567 INFO:teuthology.orchestra.run.smithi083.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 5 +430us[ +430us] +/- 71ms 2024-09-14T03:41:13.567 INFO:teuthology.orchestra.run.smithi083.stdout:^? hv04.front.sepia.ceph.com 0 8 0 - +0ns[ +0ns] +/- 0ns 2024-09-14T03:41:13.568 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-09-14T03:41:13.601 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-09-14T03:41:13.601 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-09-14T03:41:13.686 DEBUG:teuthology.orchestra.run.smithi083:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-09-14T03:41:13.750 DEBUG:teuthology.orchestra.run.smithi083:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-09-14T03:41:13.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T03:41:13.778 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-09-14T03:41:13.789 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-09-14T03:41:13.859 INFO:teuthology.task.internal:Duration was 492.002986 seconds 2024-09-14T03:41:13.859 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-09-14T03:41:13.944 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-09-14T03:41:13.944 DEBUG:teuthology.orchestra.run.smithi083:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-09-14T03:41:13.982 INFO:teuthology.orchestra.run.smithi083.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-14T03:41:14.095 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-09-14T03:41:14.095 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi083.front.sepia.ceph.com 2024-09-14T03:41:14.095 DEBUG:teuthology.orchestra.run.smithi083:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-09-14T03:41:14.155 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-09-14T03:41:14.155 DEBUG:teuthology.orchestra.run.smithi083:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-09-14T03:41:14.221 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-09-14T03:41:14.221 DEBUG:teuthology.orchestra.run.smithi083:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-09-14T03:41:14.366 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-09-14T03:41:14.377 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-09-14T03:41:14.378 DEBUG:teuthology.orchestra.run.smithi083:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-09-14T03:41:14.402 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-09-14T03:41:14.443 DEBUG:teuthology.orchestra.run.smithi083:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-09-14T03:41:14.467 INFO:teuthology.orchestra.run.smithi083.stdout:kernel.core_pattern = core 2024-09-14T03:41:14.483 DEBUG:teuthology.orchestra.run.smithi083:> test -e /home/ubuntu/cephtest/archive/coredump 2024-09-14T03:41:14.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T03:41:14.537 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-09-14T03:41:14.549 INFO:teuthology.task.internal:Transferring archived files... 2024-09-14T03:41:14.550 DEBUG:teuthology.misc:Transferring archived files from smithi083:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-09-14_03:08:15-upgrade-main-distro-default-smithi/7904542/remote/smithi083 2024-09-14T03:41:14.550 DEBUG:teuthology.orchestra.run.smithi083:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-09-14T03:41:14.611 INFO:teuthology.task.internal:Removing archive directory... 2024-09-14T03:41:14.612 DEBUG:teuthology.orchestra.run.smithi083:> rm -rf -- /home/ubuntu/cephtest/archive 2024-09-14T03:41:14.660 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-09-14T03:41:14.673 INFO:teuthology.task.internal:Not uploading archives. 2024-09-14T03:41:14.673 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-09-14T03:41:14.717 INFO:teuthology.task.internal:Tidying up after the test... 2024-09-14T03:41:14.717 DEBUG:teuthology.orchestra.run.smithi083:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-09-14T03:41:14.732 INFO:teuthology.orchestra.run.smithi083.stdout: 265628 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Sep 14 03:41 /home/ubuntu/cephtest 2024-09-14T03:41:14.733 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-09-14T03:41:14.785 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-09-14T03:41:14.962 INFO:teuthology.run:Summary data: description: upgrade/cephfs/nofs/{bluestore-bitmap centos_9.stream conf/{client mds mgr mon osd} no-mds-cluster overrides/{ignorelist_health ignorelist_wrongly_marked_down pg-warn pg_health} tasks/{0-from/squid 1-upgrade}} duration: 492.0029857158661 flavor: default owner: scheduled_teuthology@teuthology success: true 2024-09-14T03:41:14.963 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-09-14T03:41:15.050 INFO:teuthology.run:pass